{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 集成学习解决分类问题"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import datasets\n",
    "\n",
    "X, y = datasets.make_moons(n_samples=500, noise=0.3, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD8CAYAAACfF6SlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJztnXucHNV153+nWz1SD2Rn9HKERpKlZFkcCEKyHsErrW3AvIwRA5gRJk7sxLbwYptYzgrE2iuE4th67Ab8Yh0Z8wEn2GgwYhgMfAQGEiI52IysF8LW8gpoZsAgxMhI09I8+u4f1dVTXXXvrVuP7q7uPt/PRx9NV1dX3arqPvfc8yQhBBiGYZjGIlXtATAMwzCVh4U/wzBMA8LCn2EYpgFh4c8wDNOAsPBnGIZpQFj4MwzDNCAs/BmGYRoQFv4MwzANCAt/hmGYBmRctQegYsqUKWL27NnVHgbDMExNsXPnzkNCiKl++yVW+M+ePRs9PT3VHgbDMExNQUSvmuzHZh+GYZgGhIU/wzBMA8LCn2EYpgFh4c8wDNOAsPBnGIZpQFj4MwzDNCCJDfVk4qFrVx82bTuA/oEcprdmserC09A+v63aw2IYpsqw8K9junb14aat+5AbHgUA9A3kcNPWfQDAEwDDNDhs9qljNm07UBT8NrnhUWzadqBKI2IYJimw8K9j+gdygbYzDNM4sPCvY6a3ZgNtZximcWDhX8esuvA0ZDPpkm3ZTBqrLjytSiNiGCYpsMO3jrGduhztwzCMGxb+dU77/DYW9gzDeGDhzzAB4LwJpl5g4c8whnDeBFNPsMOXYQzhvAmmnmDNn2E0OM08QrEP500wtQgLfyaRJMG27jbzqOC8CaYWYeHPJI64bOtRJxCZmccN500wtQrb/JnEEYdt3Z5A+grmGnsC6drVZ3wMnTmHALS1ZvHNK85kZy9Tk7DmzySOOGoS6SYQU2E9vTWLPsk521qz2LH6XOOxMEwSYc2fSRxx1CSKYwLh8hhMPROL8CeiO4noTSJ6TvH+h4noCBHtLvxbE8d5mfokDqEbxwTSPr8NVy5oQ5oIAJAmwpULOGOaqQ/i0vzvAnCRzz7/JoSYV/i3LqbzMnVI+/w2fPOKM9HWmg1tW49jAuna1Yf7d/ZhVFhBnqNC4P6dfYH8BgyTVGKx+Qshniai2XEci2GA6DWJ4ihqF4ffgGGSSiUdvh8goj0A+gH8DyHEfvcORLQCwAoAmDVrVgWHxtQjfhOIXygoN8Nh6plKCf9fA3ivEOIoEX0UQBeAU907CSE2A9gMAAsXLlQlVDI1ShISt5xj8cslUEX7cFIXUw9UJNpHCPF7IcTRwt+PAMgQ0ZRKnJtJBnHE3Xft6sOS9U9izuqHsWT9k5Fs7ya5BBztw9QzFdH8iWgagN8JIQQRLYY16bxdiXMz4YhbS49qP+/a1YdVP92D4VFrQdg3kMOqn+4BEK6ipolJh5vhMPVMLMKfiH4C4MMAphBRL4CbAWQAQAjxfQAfB/DfiWgEQA7A1UIINusklHKULo5qP7/lof1FwW8zPCpwy0P7Q43J1KTDzXCYeiWuaJ9P+Lz/XQDfjeNcTPkpR5RLVPv5O4PDgbb7serC0zxF29ikwzQSnOHLeChHlEvS7Odx5BLUInH6TZjahmv7MB7KEeUS1X7ems1gIOfV8luzmUhjqndh74Q7kTFOWPgzHsplEokibNcuOwOr7tuD4fyY3T+TIqxddkakMTUSnLTGOGHhX0NUKk4+iVEufmOKem+SlINQLjhpjXFCSQ26Wbhwoejp6an2MBKDrKtUNpOuazu1qUCW3RsCIGDZ8v0EeaPc2yXrn+QS1Q0AEe0UQiz02481/zITl0bZaEt2P/u0876miIrF12zsVyZ27Ua5txzhxDhh4V9G4nSw1cqSvRKTHYCS++oW/G78BHmt3NuoJNGcx1QPFv5lJE6NshbqzFRqsjPprWt6PKA27m1cNFqEE6OG4/zLSJwaZdLi5AFvzPja7v2Re+/atDbLQzhbmzOh7p9OkCfx3jJMuWHNv4zEqVGWc8kexlQj0/JVhBHWKkuOEOr7mi7Y/m1nr42fIGdzCNOIsPAvI3E72MqxZA9rqgliegkz2R2RJHTZ29cuO0MbnRNmMmNzCNNosPAvI7WgUYb1S5hq82EnO92qye++RhHkjRDvzzAAC/+yk3SN0sQvIROIKuE8sTmD5qZxkYWn36rJfV9t/0OU83L5A6aRYOHf4Pj5JVQC8coFbbh/Z59HON986RmxCMogq6awQts9qR07MdIQ8f4MA3CGb8Pjl92qywpddeFpiTCRhMlclV23jrbWbNWvk2FM4Axfxgg/DVtnFopievGzrQexvasijXR+iSAOa3Kcg01BTL3Awp/R+iVMw1WDmF5MSje431+5ZTd6Xj2Mr7ef6TmWO7RTNUYnpg5r2bHZFMTUA5zkxZTgTtw6531TjRKgTBqim+4re18AuOeZ1zzNRzZtOyAV/ARoo4x0E0OaqNjgRWUUrbfSD0zjwcK/jojapcnWuPsGchCwNO77d/bhygVtvh2vgmQzq/a1Vxiq9wXgmUxUJh8/T5ZuYsgLgVfWX4Idq8/FRE2mMcPUMmz2qRPiCFNUaeRP/fYt35K/QbKZVfsCwNe69qG1OaPszeueGNKSip42uutvn9+Gtd37pd3BnGPWZRozTC3Dmn+dEMTsoiKM49QmSH2cVReeBlIc555nXsNxjSPWPZnoKnrmhkfxt517iisg98roY2ed4jtmXaYxw9QyLPzrhKhF5GzHqQyT8gxBGqK3z29TmmUEgNxwXnkeWzDbgtyPUSFw09Z9+FrXPqVJy9kHeEKm9CehuvZ6rPjJNBYs/OuEqEIqrOPUJkjXrXm3PGY0Jjet2UxJNJCumJyT3PAofvLLg9KV0cN7X8eJkbHJ5p3BYdy0dV9xtcAVP5l6JRbhT0R3EtGbRPSc4n0iom8T0YtEtJeI3h/HeZkxogopnZPVtMKnU6t2ClD3fjI7u5OJzRnptdjN2sPU81eZh94ZHNaay4KsaJiA7O0Ebv1TYG2r9f/ezuoco0GJy+F7F4DvAviR4v2LAZxa+PdnAP5v4X8mJqIWkdOVSe7a1Re4wqcsFj6I0L5yQRue+u1bgRLP7PH6dfYywXmOpNdnqkn2dgJd1wH5giJw5KD1GgDmdpgf46HrgeHc2DEeuj7YMRqYWIS/EOJpIpqt2eUyAD8SVi2JZ4iolYhOEUK8Hsf5GYsoQkpWSA0Ys5nbx5dh6m8w9T+8MziM+3f2KTXslmxGunpozWak5Z5VZDNpjB+X8o34ccOVP2Pg0RvHBL9Nftjabiq4n1g3JvhthnPWdhb+vlQq1LMNwEHH697CNhb+CcEWXn/bucejOdtRMzamFT7dAlQX4ulGl0VLCs80UekKSHcuuzYRAG31UDdxFZGLMmHUxeSTOxxsu4wjvcG2MyVUyuEr+7l61uZEtIKIeoio56233qrAsBgn7fPbkFeYTEaFwKqf7sGq+/Z4bPumWcAyv4QO1UphQJED8M7gcDECaMfqc9Gm0N7tgm/2SimITT9MSK3KJ/K1rn2Bk/JM/SsNQcuMYNuZEiql+fcCmOl4PQNAv3snIcRmAJsBq6pnZYZWHmpVO9Np58Oj3kdiJ4F984ozfa/X7Zfwe8Aq04vK7AOUauKmndT8zGXOZxmm3INqwrjnmdeKx4vSQa0maw1lJ8m1/Owk82Oct6bU5g8Amay1nfGlUsK/G8AXieheWI7eI/Vs74+abVutiaNrVx8Gh0YCf05W4VOFc795tzymFOI604vK7GNjC0M7KznKvTQt/RymiFyQgnH2dyJIIl6iFZCLNwAPfgEYHRrblm6ytpti2/WfWGeZelpmWIKf7f1GxCL8iegnAD4MYAoR9QK4GUAGAIQQ3wfwCICPAngRwCCAv4rjvEklinYW1abcN5ArRry0BfjBB61v7yRswtPaZWdg1X17MJwvFYMTmzPapjAqs48TWxhGjdQxiVDyC6kN4utQCXG/Z5MiwpzVDxeFPIDI5T7KSlyCe24HC/uQxBXt8wmf9wWAL8RxrlogSrZtmInDLRxsh22QH7yJkMukCRAoEdZRG9Lb5w6inZoI07gycP2emckEu+rC06STnAzZuE2ejfuZjx+XSr55iAV3VeHCbmUgSJEzoDw2ZRvTH7yfkLO1cft8cZkSVJq5zmShCku1iTMDVzfREOApeKca9y0P7VcWq/Mbd9C8htzwqPLecClqBXs7G858xMLfkCD2U1NHo33cctmUTd+3j6/Tpo8X6u0ENaOEsTv7mb7cK4aWbAZEljmotTkDIYCVW3Zj07YDkSenVReehpVbdhs1i9GNW2eqosKxVGNVPRu7tWQQuCaRhAZNFms44V8OYeQmiDmjEjZltz3Yro/jHN8575vqacjuxL2CMLmPYfwXXbv6lLkGzvPbk4BzHK3NGRw9PlI0r8Rh526f34aeVw+XROYAwRva6AS4rs+w7cdxdxSzz69yAk9szuD4cN44f6GhiSNZrAZXDg3VwN2vWbmKMA3CTZmz+mGlqcdPI9QJBxXZTBpXLmjzCPpsJo33z2rBL146rB3PrcvnSevg28d1lmQYHBqRmjpasxnsvvkC6fXoVkEE4JX1lxjvb+P3nLp29ZWYZexM4aA9hVXP0r5vQb57smuzn3GbaxJXHReI10RXt6xthfzXQ8DaAf/Pu1cOgBVyeum3qzIBcAN3CWGjcKKWS9YRRiMEvMJBYEw4pAhQ+RbtCpcyzfqZl9/RTiAt2YxS2OaGR/HPz7xWfK1biQzkhqX1gm55aL9WkLtNFqa1gnRj6drVh1U/3VOSwzCQG8aq+6yMZvdKQ3UMVVVUe9xBnduqVpbu74XfcVnYG9AywzL1yLabUKNlJhpK+IcV4kEduEEI4h9wohIOJzWlkRfQCkVV4TNdQbRsJg0i/XGD4J5wv9a1T+sQld0T08k3rUkM2LTtgDR5bTgvQoXmunGOO4i/JMh3NYzTnHEQNVmsRstMNFQ9/7A178tZ0z1syWCVcDg2pI70sFEJQ932b15xplF8vZNlqe3Y3nQ9Xh5/DbY3XY9lqe3F95zj79rVh3scqwbV+d33xHTyHRVCWf5AN4GEDc114m4OY0rU/gxcBiIAczssE03LTABk/e9nsnGWkibFM054mYmG0vzDatlRyyX7IdPc/LS2IIlDTnQ2f9V2W/D6FUtzsiy1Heszd6CZrAzOGXQI6zN3AMNAd35piRDTmUwA4P90nCW9134hn05Ujl/dfTQRtH4ThN0cRnZuGSZOXhPqpgxEpQiSc+C28QvJ968Gykw0lOYfpTFH+/w27Fh9Ll5Zf0mxKFi5MNHawqw6WrMZTMikcM8zr2FCJoXWbKbkPny9/UzcMmc/doy3tPUd46/HLXP2l8TXywqzpSQLhhvGdRYFv00zDeGGcZ2e8esEqN29S4b7ebZmM1YimgRV8bVVF56m/MyxEyO+mrLJBGHaS9ndocz24wDBm8iU00/V8Mhs/ABAaRivHBJAQ2n+QG005jDR2trnt0mjbgBINUa3Vv/O4DCymTRuXT6veMxnu/8Rl766HtmC0G7DIVz66no8230yFi27VrkCWrllt2cM0+mQ9Nqm09vF8Re3KbRvAordu1S4n2fXrj58WTIeQG0vByBNwhrISbR2V0jfbad/CX/57Ht9Vx9hTUgyJ68JJn4q9gmERGXLF3mz6KCE0FCaf61gqrV97KxTPLWys5k0/vzsWZ7VzVO/fcu3FHHbrzcWBX/xeDSEmb/eVHwtWwHJtN9+MUV+DWIyJjZnig3Y56x+GMdOjHi0bwLw52fPCiyM2ue3KUs5q7T09vlt2LXmAunnSu6Rvdw/chCAAI4cxKJ9N+OWOfu1TmXduZ3Eqa37+anYJxCBOiklzcI/gZg4+7p29eH+nX0lGj7Ban/49fYzPQLaT7B07erDNCHX1t+j2G4jEzS34WqcoPEl2wZFEzaOdOD0U/6gRPAM5IYBYSUm2RPWrcvn4evtVqx60D6tKvPU4JDejOMrfBUhfUtevd03UsrETBfVyevEz8QZtC+Bc7I27T0QiST35j1vjWXTd1IDNn43DWf2KQdxL59NHNMqE8FTv5U3wfEzA2zadgALxRTMkJhr3qQpmKYZr8wctPTC6/DNh8fhs0P/jOn0NvrFZGwc6UB3fina/+MhPJ7egunjD6FfTClub24ah11rXMlfezsx8uCXMG70uPX6yEHrNaC0qdrjcZvF/JyvvqYSxXL/FLzt2ZYmQl6IQN+HsAEJKnQmziCrjKglyj34ZcMmvdxCnZSSZuEfkdh/GDCLLgpqIpAJFiqM185g3pjqKInQASxt/eCCVVrhb4/Zfb0rtyzGXVhcsm1Zaju+MU4eBfTQwFLPcQcfXYNmW/AXGDd63Nqu+bHZ0Ulun4gu4sVX+CqSgfrFZM+2vBAl2cgq3IqDnSVtl+Z2auNx2uOD5K7EGjlkIthrIWmqDiqSstknImHa+pngF10U1ETgNAMApU5hO6ywO78Uq4c/i978FOQFoTc/BX9Hn8eiZdeGugbZWHRRQLL9J+TekB5btb3I3k5sGfycb46BE99oMMlyP4fx2DjiFQImppquXX2etphbfnWw2BbTXaY5TlNLkNyVWCOHdILdpkaTpmoN1vwjoooRDxODH4QwJgJbO5fVKrIngu78UnQPWRp4JkXYdNVZsY7RjvZxM53exjnvm+rZ3p+fjBkprymqPz8ZSvdaQbuckbKuUZdj4EYbDSZZ7j/3x1/C48++F8gHN9Ws7d7vqfE/nBeeInKApVB82adSaRDzY5DclVgz3E0Ee9RyC4wRLPwjIqunbm+Piu7HHCXxzFhji3gJsjEep2loznk7ePaLyVJ/xR1Nn8QNw7d7TFF3NH0Sa6G4R//i1S7t1cXj4kPRMrPndqBrdIl1zt/lMP1EFlcumFpS0O6c903Fpm0HsHLLbu1zUbWw1CW8qcyKYcyPpmHPsfoiTAQ79+atCA1V1bMczF79sPK925bPk9acNxHUqpoxfi0OTVBVKZURR+XSEvZ2YvD+L3iE+erhz+Kh/FKPnbxrVx+2P3A7vox7i47j23A1ll5+HQBIhdLz6atBEhGaB6H7sv36e+dwRg5mp2Hj8HLcfXSxsj2ifU7bPBSkcqzuu+OH+7mUs/IsEGNQg2kFzFopkZzAcXJVzwrRplgST2wurYDp1PJMtDJVzZig5QJkBCmLEHtG6NwObOzeL40CksXZW9d4HZZvO88jeJasf1Lqb+kjuakolZ3oL/gdgqk59zpuELfjcGoI3QNLcdPWfZiQ0bdHDOIcndickRazMynO1zeQK6mMWu6M3tiSI00jZWyHqi1ct66w/k+AcC2S9KgkH1j4R0S1JBY+P16/aAndjzZqjRaZOUZVe78cnZ/mXbIC52/9AHJDZmYEleBR3aONIx3435nNaKKR0jdOvGv9YFU/TIkz0jYXdQ8tNWqPGEQI33zpGZ5y0pk04e8vH6vFr1uhOZWAclaeDY1KKzaNlEm6cK2FqCQNHO0TEVWEyBGFPdeJTsD7/WijanTuaKJL5sqzhcvR+SlKjSUnqnvUnV+Kd8UE7xv54WJUiSxpSSickSontWwsqjG1Nmc852uf34ZNHz+r5D5s+vhZxclux+pzcdvyedKENaA0qkwXvVPxBC1Amg2Nh64PlqxlEhlUTWo8KqnuNP9q1CuRaaYmFTBlgsK0O1ecGp0uW7hcdeLjMCPozFcT6aj8Q0d6lc7RRanJaJMkuREEtjddj40jHXh6/Dk4MaJujygbUyZNOHp8bGWl6kksw37Pr16RKgAAQOx5KEbEoRUnXbjWeFRSXWn+SapXoioxYCPTqmVVHU0/K8NU4wuSLVzVe+xK+W9P7yjJXXCiqi2ElhlKu/yG4Q4MiibPR4iAGalD2JC5Az+Y/4p21SJb1ZzUNM4T0hkkF8S0XpEsN6RceSi+xCG4k15Dp8bLPMSi+RPRRQC+BSAN4A4hxHrX+58GsAmALSG+K4S4I45zO0lSDXO3JmYS7aNy8k5szqC5aVygUMIgoX9B7NSVusfu1cVtp7+ARftu9th/2y/9NtpXd3iud+NIBzZk7igtVFf4Yfb/WH693fmlwLCViNZGh+CO1s3SEBa99J2SCqcy3Nr8HEVUj5/pznkPWgrlqp3+ARMloGqlnePQipMe8lnjZR4iC38iSgP4HoDzAfQCeJaIuoUQz7t23SKE+GLU8+lIWg3zoKYN1TgHBoeLNW9MhXoQIR3EWViJeyy7xuk7NwKkNiO4J9ud/+l8PHf6bCx66TulYZs/PgkpRW4GMJbk9vL4a+RpDiFMDmGcse57MJAbRiZFmNicMQ4XDnvuWIhDcNeCcK3hMg9xaP6LAbwohHgZAIjoXgCXAXAL/7KTyIiHAKjGnyLCnNUPY3prFgODQ0ZCXSWM7Vo+TsFxzvumerJKVVplJe6xbOI6BYrKog5hXDLZ7u0EnhgT/GuOXYmfDll1hnQVOG36FUXuwpgcbD/A+aP/apWxoEN4HVPQf/oNAEpj8J0+HzfDeSEtfqfzwaj8Iu8cO1H8TpXFLxaX4A4jXBMYe59E4rD5twFwru96C9vcXElEe4nop0Q0M4bzeihnr91KoPITjApRtK8fG9KHGtrohLHTTh/U2VuJeyybuHQ2fA+uSJPm3OtYR5tL6vsA+izsjSNe+38O40OZHNrnt+FHi17FhqYfYkbqEFIEtNEhy4zliH5x+3xkuO+Nnw+mfX4brlzQ5lnFDA7ny++zmdsBrHzOanCy8rnKCOAwUUZJLh9dRuIQ/rJfkFu1egjAbCHEXAA/B3C39EBEK4ioh4h63npLXppYR1whhNXCPf4gJSLcwt7P4WyvFoKWhja+xxF+ULKJa+NIhyV8najMCJp4fSd5IXDb8nnSNo6yInerhz4TWoAteuk7yOJE6UZX2KJfM3jAe29MHLpP/fYtbcmI2B3A1RSmQcND4whJrVHiMPv0AnBq8jMA9Dt3EEI4A6V/AGCD7EBCiM0ANgNWeYcwg4ktE7FKOMevchTKcGveThu4SpPU2el17/ne44jJOTJTxePpD+Ev3j9mw9cu5xV2+TY6hJfHX1PsIbDzP51fvI6VnbvhtgY5i9wBUEbcGGEQ/eLnN5GtsEyerYk/JjafTbUTs4JGGdV4olYU4hD+zwI4lYjmwIrmuRrANc4diOgUIYRdzWsZgN/EcN6KU+4cAvfxW7IZZfEvJ63ZDACrvkv/QA6tzRkIARzJWY5BVRkBXWZviqikfEAgIv6gVDHri+ZfBMCgvLQi0oTIWqbOICts87nTZwOwQiJ7Xj0sraZZ/Cy8E6wT3++GQfSLyp8CWBOP+5hdu/qUuSDOFYLuuLL9tfjZ06stTE2ijJzXoHriScklKCORhb8QYoSIvghgG6xQzzuFEPuJaB2AHiFEN4DriWgZgBEAhwF8Oup5K02Q0Mkwk4Ts+Jk0IZMiT4y4k0yKMDyaL0kCcgrzvoEcMimShgme876p2PIryQ8Flp8hdDJQDDHekVZwskgTF3bYJnCt1O/hRkB9H4y+GwbRL6pSISrT5aZtB6Rjdk9UMoe+E2OfjYlWX+3ELL/7LCssJyMpuQRlJJY4fyHEIwAecW1b4/j7JgA3xXGuamEaOuknCFQTg+z4w6NCGuNvlw9uyWZwbGhE6QQuHicv0JrN4KTx40rOu2nbAe3EEjp+v9qZj+5IEx/tzsTW7jT5uJ/h4NCI/3ejMKbBR9dgQu4N9Ocn4w7xScwbXYL2wmfsFchPfnkQo0IgTaR0vANqU41zolJNbM2ZFHLD+WArWBOtPmnP3r06kV2DmyTlEpSRuivvUC5M49t1k0TPq4fxz8+8VtzeN5DDqvv2aI/vjPF3s2T9k0ZmIcAyAe2+ufQ4KxUlA5yEsgVXOznHbZoYOgbkDnv3KwikILZ22eSuwhOZM7oENx29bez7MQRkXYrB/Tv7iqGoo8Jq7AJgrJm9A5U5xzlRKRMHTxqP54OWejbR6iv17HXmJ114qGxiKkINFRpaV+Udyolp20RdfL1T8NsM5wXWdu8P3JbRPqYpsuOY2HntfYyLg9k/yuEcQIVoo5aZ3nrt5UIWvTF0FEhlSvdzCCTdfXBHM5msEmyCRuaoIq/ueeY16f02CbuNMylvMCvv5FyyfW6H9axbZsISpmV49mEjdPZ2QtmhqGVmZUNSEwALf0NM49tVgkQXtjmQGw4cP287+0xQHccvHNS2HRvX8yn5UQIQo2NCNsgPKkqooGxZPzpkVfS0yU4qEUiqe3/b8nme/smmQlN2z/0Esc6MIwvFNAm7DaNUqNg4vNyT+zAomrBxeHnpjuWO7w9b7fOJdZCbAKkhzDxuWPgbYhrfrhIkJlmlQXIUVM4+wLLntmYzvsexr2lic8bzHgH487NnBSsOFkcJ3qhJOtplfYGR0jHaiVD2BK2ztTuF5rLUdmxvuh4vj78Gv5hwPT598q+091wlcFsK0Vo6gaxrOu8u5mbTtasPg0Mjns+ETcq7++hib+7D8Gdx99HFgY8VibBOZeX7omG0fSds8w+ASQSKKkzRr8Tzpm0HPD9eHToN9Pm/u9joGPZ4dY5o3bk821WCN0ikR9BQQdPoDdfxBrZ+BU2PrkFz7g0MZqfh+LErMSr+KwDL1n7/zj4sfO8k6eRul2pYn7mj2I5yOg5hLf0j1l5zBjB3rBWl8762NmeQApB3DefY0Ai6dvVh1YWnYeWW3b6hmyao2oC2ZjNYuyxcG9DprVl0D5TmPgAR8x+cmJZlCOtUVn6uLAUHEg8L/zKgmiTcXZucBLXBmjj7goScysZsf96op0DRnirZO0ikRxxJOga0iHdBuXcBoFj+YSiVtyp7Qh3pZL8++8EvohlDpQd1TVJuASzLpwCsqC578pflG4TR1FW+iZPGjwsdQhtrI3c3JmGkxcnhIDzfNROnchhndB3XCWKzT4WwuzalFIb6oJqdn48gat19vzoznh99XPbUoDXcQ8aPu10wzTSEm8f9qGSbztQyzaDQ3KZtB3D+6L8WTUPbm6731Bdyn+vr7Wfi1uXzjMx/Oido63eZAAAgAElEQVR8OaqvlrV8ip/J0O1PgkDReWvqVA7qjK7z0g+s+VcQ+0cSh/bkNi/ZWb0rt+zGpm0HcOyEQey5Bl1Ui51tCoxlFb80oVehSQS0pwbVzlRLeQ1CeIU/AEyio1iW2l7U/rUTsuK8b2AKnilkRi/4/eMlpqEZdAjrM3cAwyiew8bdlCVMUqCqp++y1PZiNdE3aSqw91ho7bVs5VP8VnzSFZ6wIsqCaONBqoRWO1u5zLDmX2Hi1J5sZ9+ty+fh+HAeA7nhopaviv831fxU+9kRQGu79+PLW3YXVxb9+cnyAwW1p5pqZ7aTt2gCcI9STV7xPhGKxd98J2RJF6dB0YRvDF2Fm7buw9e69uGGcZ1FwW8jKzDnPpdJWK2fE95eGS5Lbcf6zB3FaqLT8BZGHvxS8rTX7ET9dtXkIEbLp41XO1u5zLDmXwXi1p6ixJ7r9pOZfFqbM2O16ZssbbJfTMET+Xm4ip4uFXZhk3v8tDOPk9c2AQhrsjj1AmDPj6W+gEHRhPtGP4i/TP9cqv1Pp7eldXSkYwTwxtb/ifeIQ+gXk7FxpMPS6POj+MkvD2Jdk9w01JayziHzxcg0+pVbdqPn1cMliV5+Zh37eIsf/ILHNzFu9DgGH12D5kprrzr7+cgJ+WdOHLUmeV3xjXJp436O5Rr3B7Dwr2F0jT9kBDEvqZx7QsAT6TKDDuEqehr3jX4Q56V2Y3rqbRzPTkPzxWVaHqtMAC0zrbhyAJh19phzkNKAGMVg9hSrm9eJxbhs3C/Rinc9h061zsCOla7MV9WPfG4HPvDjk6RiaVQIZUMYapGco4Au0csZfaRrquN09L80/pB0ITQh94b0/MYEFXx+Dt3hY/LP5YfMzHrl0MZ1JshqVy+NARIG8efVYOHChaKnp6faw0gsqlA+J+66QKpev6qIINl7K7fsxr81XY8ZKa9QGxEpfGX48+jOL9UWJANgJjxU+6xthdK5vHZAd9tKjy37YbtNTLL90k1A08lA7h28gSn4xtBVHht+mgiX0L+VTJKA1RAme8V3lQJizuqHlTpuW2sWOwolGWTPP5tJ48oFbbh/Z19x+3bFs+rNT8GMdS8pzuSD6t6ddQ3wwmPyZ1o00bmwJ+y1LeHG4j6ObKxRtHPV5/2up4oQ0U4hxEK//Vjzr1H8TD3ZTBo3X6qP5/ZzGsrMU5u2HcD0nNycMY7yYw7N4aVq57JpWJ9qnziKh5m2GVRlDBdqBU3DW9jgcuKOCeEPFRvCT6e38Tomo3/BDVjkOEeQMt5OU48un8T5vdg40uGZgAZFEzaOdOChsG0cVY7QnjtRnJTdz9TPfp6dJK+/ZILKvBiHdq4yQdaBP4A1/xpltqbRi5HNGlakjipXYIei6FfXrj4s6vog2mT9bQv05qdg6dC3QQBeWX+JdwcTrUm3j2o5Xo76QcpVRilvYCo+cPxbJcLUL89Cpr27S2870T0XG9nKYSza5+1S30QB31WamyBauv1MVc/T3ufUC4Cdd1kOXD+yk4Cmk/y1+XJq56z5M5XGFigq0kQeAaESQmFiwdvnt+HZgzdg0q//l7ctYYE2OoRfj1+BVjoG3Cr5cZpoTbp94moOboJhKOk0HJJPdBpUZbxPakpjcGhUmejlfp7OMt8pIk8pEXdHMjeBS3cXfChG2M9R12PhyEHLQZ+ZYFVg1ZHJAhdv0MfmV6JRS7Ur18YAC/8awsTO7/7h60w7OqehjkXLrgVmTwQe+LxUCBABk3DUeiFbapuYbfz2CRKvHQWDxjAAPKGKJg1eVJPs4NAobl0+Tzphy47rrBZrUkNKRqDkL1PBD5Q+LwB49Ea5ecckS9te9c3tsIS881jZScAZlyujvKRjCoLM9n/ptznah6kMQZuOqD5ja3qR0vXtL7mJYHSH4ploTUnRrNyrjOxE4PgRrwA88a4lIAr7mzT/0U2+qnDgIGG9QQiUYd4yUzox5wVKM9hlz2skwCRjk50E3PjK2Ou9ncCDX7B8Lza5w0DPDw0OFqKCp8x3sPVz1rh0q5CEw0leNUSYBt86006khDNn3X4TnEttk0SuStSFd6BNrHKWKL7xFWCCxOadHy6pXmpiUtOV6FCNJ4iG7qw66iwtEaR0uBRFgts/jX7Eqvipel5+35fsJM9xi2YeJ0+sKxX8gQhRwVM17tzhmi73wJp/DRG0wbfuM7amFyrhLEwlTfdSW2e2cS+xr9hcVu0qSH9mAEDuHfmBHBOciUlNFbEDILCpzk17ege+Mc5bWmJSpgnzLlkRuMd0Cc6WlINvlDiR12fS+OZlCgVCZ2t3Cnk/U0oUm32YCp6689VwuQcW/jVE0Abfus9EqsQYtJJmEHNNFZJnTPszFzHwWZjed9nku2T9k4FMdW6ymTS+2nQfmkclpSUyW9A8/xZ1CK6pDXtuB5rndpQ4n32jzFT3jdKlqwS/56x1wisqywLhzYZ+Tv8aCu90wmafGiKMmaYslRi1X3aylu/ZSQhlromjIUxAAkc9ScwebsES5b4HNdV98uxZJa+vXNCGySNvSY/RrMrsDVnBUtdMxoPqvl3+/eDF/9JN3u2pDLDwr8dMhVG+h37jdlKp5vQxw5p/jRHGTBN7JUZdU4yoMc5VSJ4JHPVkGGrant6B9vHrgAm9wPgZQHoNAH/hE9VUt2T9k+gXk6WlJZSCqhIVLOMK0ZVFDgVxvgbN+tVFKtVYeKcTFv5McOKKxJH9CKNm74ZI5w9lGnP5LLp29WFTobz19NYsbjv9BSzad3Mo81VUU13/QA4bU/LM3mbVMwoz6Qa51z/7ylgSF6UtDf1j/2B0PVLChvqGNSva56vxYm5OYjH7ENFFRHSAiF4kotWS98cT0ZbC+78kotlxnJepEnFE4qjMDKdeYC3fnaQyZhOL7JhbV1gZqZpG8FFNY7LGOdN3bgxtvoo6numtWXTnl3r67W7MXKd+RkGb6AQxE/3sK1YYph0eK0at1z/7itH1xEpUs2K5m9NXkMjlHYgoDeD/ATgfQC+AZwF8QgjxvGOf6wDMFUJ8noiuBnC5EGK57riNUt4hSKvFuhqHKj0+OwkYOloaypduAi77nrfgmlsDK7b4U1CmEhCyMhkvj79G0bUtQPG5kKiKvvkW2gtSMsOvXINTI75lkjwxjNLAzSHq+UTRvuMoCphwKlneYTGAF4UQLxdOfC+AywA879jnMgBrC3//FMB3iYhEUgsLVYjAIYaa40QR3HGNIxAqc4Is+3N0qNT2rFq6B0k2i3H5LnPQqso5m5qvVM/U5FmrQkhN+hMY3xOdOchtSlFlBAfJFLaJGg0WR1HAclAFc1Icwr8NgPNu9gL4M9U+QogRIjoCYDKgaoTaGAQOMZQQh+COYxyBCdp+0SlsVEt30+PEHE4qc9BuHOnAhqYfltY/MvSLqJ5pz6uHS8o16551KCd/EDu63/NzTrSqWkCU9m7zQ/XsH/i8ZeLzE5xJyRx3UqXeAHHY/GWLW7dGb7IPiGgFEfUQUc9bb8lD1eqJOJps+7Xzq9Q4AqMK+8tOku/v1MwiJfnMiD2cVJap+3j6Q3ju/X8HtMyEAOENTMXfHPsrLHlkirQtoxPVM/3JLw9Gftax4Rf+CFhCbG2rVbBNxoJPBz+vrp2jSYhqhTPHjahCeDMQj/DvBeBMm5sBoF+1DxGNA9ACwLO+F0JsFkIsFEIsnDp1agxDSzaqUMIgdVbiENxxjCMwqh/hxRt8Y+jNl+guncM+TszhpCoH7aJl16Lrw9tw+ui9OPv4t/BgfikW/P5xLOr6IMTaVqUTWvXsVEXbyjpJA2P9kp1jLnl+OoRVqTOVRvF5UBpY+Jlw0T4mz95PcNpO2ys2W6+3rtAGBJSdKvUGiMPs8yyAU4loDoA+AFcDuMa1TzeATwH4dwAfB/Bkku39lXJ+xpF9axKj7nc957xvKu555jVlCeGyoTMz6OyfRpU2yfpxy46jcgzrIlv8YvoNCrHZzdSL4ZeK5b3qmaYl5Zrt/cuGn0nC9p/4PY/8aDx5IOetsYS1X48FP8GZpDaMVfJDRBb+BRv+FwFsA5AGcKcQYj8RrQPQI4ToBvBDAP9ERC/C0vivjnreclFJ56d9vLXd+4vdmyZkgi3G/CYQv+vp2tWH+3f2lfyUCMCVCxQ247gdU5r+uEpKnJOqiJMZ6uP49WZ1jsfdDD6gkHBq5TeM6yxtcA9IE6lUz/SWOfux5NXbcQoOoV9MwcaRDjye/lDJJB274mKS/OV2Fpezjv7cDquiph9+grMSSW2mVMkPEUucvxDiESHEfxFC/LEQ4u8L29YUBD+EEMeFEFcJIf6zEGKxHRmUROKwoQflxEi++Pc7g8O4aes+X7uwjV9MuN/1qBqGP/Vbic8lZAkAJVGOV1y6/8DfTCT77FnXjDkcKW29Brzj6bkzkj3WqZVPV3U/cwlF2TP90aJX0fH6JrTRIaQImJE6hA1NP8SPFr1a0nPZnW8Q5LtkMjbldmf8u8oUFJcm62tqImvS1pGkNoxV8kNwhq+LSjs/44i00UV2+F1PoOuNW1uK43iyEMVTL7Beq6I/9nZa2rwz6WjPj4H9D0hMFyot9qBlJ/ZZ+dx2+guYvnMjTsEh5JFCCnnvTpQq6QUASJ7prV6zShYnsOil7wC4FkDE75JqBRbGJCE1yxkIZFN8zX7Cep6zztYntSUp5LNSzYkccGE3F5V2fpZ7svG7nkDX66ctyRyDOuLSvpxa53lrrB++O8v37mVjY3vg8/JJJ2gDcb+Vyt5OLNp3c1FbH0d5+VQiRv1XPAb3KvR3SbcCMyhi58FeWZU43AsCOQ6nqq0pqyLDAP/VWZjrAoJ/xxMMC38XugYbcdO1qw8pkqaBxjbZ+F1PoOvVlQAIY8IJWlLABGm5aQG88q9jYwucXCR/RgD0QkYyFuWR/ISVwb2yvzPuJi6fOvlXxX2kTWIevVG/AgtjknjhMXhWTc5rjCpE53ZYTdx16JSIMNcVt9mzyrDZx0Wo7MgQ2PZZWfRGnJON3/UEul6dYyqMCaccjq7INltXPfhM1tJiX3hM7VyOawWj29/gXq268DRsf+B2rKPSJi5fE98H9p6BrtElHuf/9gdux2Wpw/JJyR5PGJOE7p7EFWnjd3/9lIig15UkJ3EMsPCXEGcJZFX0haoXa5ooer19F37XY3y9uhIAW1fIP+OnfamOF5agmcMe3Nrqcev/lc+p69notHJV8xLZ6kMnrAzuVfv8Nlzw2P1ozpVGFI0bPQ48sQ6bTnzb8537Mu5Vr0airMB0NnWVEH30xmDPXvesZUpE1Ei1JDmJY4CFfxnRhVmq7LB5IapS2M0YlbYU1oEWt6PLNA4cUAvhEsRYY/CgKxXV/mddUxo+6nccG4PWl8251+XvH+lF/3FJPogqAskev+ZcWiGqu1cqRSF32Kr0aZr8pXL8ymr7x7HaSJqTOCJs8y8juuiLqmTVlpOwDrS4mdth1YrX2emBsQ5SfvvZ7LzL307stmMD8v0/9g/xhvbt7QQe/IJ+xdMyQ/rd6hdT5PtnJ8nHY2r31t0rnbDsudPchi47xxU/AG58xTv2OEooJOU7HhORSzqXi3oo6Txn9cOq4rG4dfm84GV3k06SGl3IkrVeeMw7Nl1pYjdrj+jPF6QkcpxsmKOPVCqMw23zB4CPN/0C6zN3WKYh1/6BSjkHyd7d26lP1IojE9hNXKWck/QdV1DJks6MAl3phSCO1qTU/PelCrHKSkzHYlQqAv4VKCM4A+3nu/D3j+OmpvvwhzgECiJYdILfUVu/vbDJ+V1aeuF1GJc+K3opZ2e4r9+x5nbIWyIWj+WYXOIStnGZbJL0HY8IC/8y4ld6wcTRWpVa+9WmktqVy5E6msoglR/yGoP8KlCGbIM4+OgaLBt8A/9NnIQ/yBxHE0YKn4up1oxLg5Z/52Io5ewM9zWxq1+8Qa/9r22xkt+EIykuyj1JYinnKsM2/zIStR0fUJ1yE1Wl3LHUqgqVK5/Ds+/fgKFRV2qSAH43+Wx/J2TINojNudeRIoHJqaNoopHSfUxt0qpkJ10SVFDs+3bkIJTVUlWrH7uV5toWy0TlymZW4hT8zuOFKXWcxFLOVYY1/zITNWy0KrX2q0k5Y6l9NNOZv96ErKvwGhEg3n7J/9hBNUtpMpoE1crBuTrKToSlxzmEZSpjaddx4PFnCBTzIZwtG1VRPE5be+6w5ZwGrM+GCcsNG1pZRyabOGDNP+HUXVSQH0HMJyZZos59VGUdCprke4S8gdB7hEHDuaCapakAk60c3Kuj3GEgPa6g6RfO3X57fIJOlTVtO2btss5kKE7stpwmDWFk1GhoZdJgzT/hxFHzv6YwdczJtPitn7MciXaMt3sfVUx/QRC/SVMxDd4J4E2agmnO86r8EXG2QQTUKweZMB4dssod3PiK2fmDYOLkfej6YGUzjvR6E9co5X+MBrfTxwlr/gknDr9BTWEaS60ym+QOj/kITE0rlALWtuLk1AkMiVJ9KCeacPD9q6wXcfojJNd5QqRxWJyMvLDaPnpWDiV2dwlBzCFBauv4+TNM77Pss86ifJd/X78SyE5qeDt9nLDmH4FKhWDGWW4i8ZiWfNAJOtuUYyoMC9rmyfnf4wQsAdyKY+gXk/GL2dehY9m1lnB84PNezdTpgPQbs3vVcNY16P3Vg5hOb6NfTMbGkQ5055cCsCzqr8y9pPSzfiGpJuaQvZ3eMEu/KBo/f0ZQG3y6Sa69u5v02BnYTr+CfQ0Jj7WvBTjJKyTuEEygDpK0agnf5CwKXeenNz8FS4e+DcBaae346CF/wZvJ6hO8FElga8W1uOvoYs/h2lqz2LH63LENftdrklDmN4Hokqt0AjdIopys9EIQqplMVyOYJnmx8A/JkvVPShO4PD9apjyYCLLz1pi1/HMhhBWf0i+mYNNIB7419SG9cFMWanMIU4WAHMyeggVHb5MrEekd/q0R7fOYaL8mQtr0WE5MViWpTDxO6DgyjOscU+HPNv+QNFwIZtLQNfSwTRJzO8wjUBwQodgqcX3TD/UCM93k60j2/O2gOfeG3KeT3lHqX1DhjLjxw8Q8E8aPURLppCA/7B+fb+KHqLPKmtWEbf4h0ZVuYCqEHV2jM0nIEoUCkMUJ9ZuUspYJKpw2eE0Uk9Snc6uBEzVo5IupGSxMXoUz0klVR8cn29koO7jOKmtWE9b8Q1LJjl+MD86IEbcWrNNGs5OAhZ8pxuYHM4ASMKHV0mhVDB0b016DVoT002QpPSakTbX0ID10o2jSYTq0mVbdrLPKmtWENf+QVKrjFxOBvZ2WAFYxkrOafBdKN1AQxyUEkHtHv4sddvraM1ZF0eGcOoLFjUrDzU6yxh2mLv0Lj+nfd58fMIuskVVQDdqvwNScM7fDup8777LuI6Wt/gh+184RQh7Y4cvUJyZOSKDUUSj9jKuto/NzgOFkIWkNGSYyJ5MFxmXl1TDd1yETdMqyxi7s8QH+kTWqcc5YDPzH9jEBveDT+vpIpo7cMNE+DRYhxA5fprEJUztHVqJh4V+rzQzG5Qk0jcxtTBvBqFYb7mxbWSKazuxil6x2lqUwMcWo9nnl6TFHuBi1VgI681SU5D6/Ym9xNHKpQyIJfyKaRESPE9ELhf8nKvYbJaLdhX/dUc7JMEaErZ3j9h+UdN1Cqa0dKBXQfjX/VeNTCWzA68sIk21rj/e8NVbIpQwxWhol5R6jauzK+2ww4TkxrY0UJtqHI4SkRNX8VwN4QghxKoAnCq9l5IQQ8wr/lkU8J8P4YxT9QZag9StvMLdjTDO1tVmZgJaWJ1C0ibTr39/6p1Yugqlm6qch6wTd3A5g/B8oL9NzTt1E87OvALdMgpEZyTkGHTrHvcmYVIT5TAMQVfhfBuDuwt93A8VmQQxTXWRCMt3kyAtw2OFNYttNTAdu7TU7Ccg0y483eNi/7+6Rg96Ydz8N2U/Q+TmpnQJaFR307utWU3tlETbNhBeVMNE+HCEkJWq0zx8KIV4HACHE60T0HsV+E4ioB8AIgPVCiC7ZTkS0AsAKAJg1a1bEoTENjaxOjF35EvA6Tf1i23UatczBCugdzsOaKKQSXGYgO55eNU6/Ojx+sf5Zh+VWFR2UH5FvB6zJKEy0jymmtZ+ifqYB8I32IaKfA2MVbR18FcDdQohWx77vCCE8dn8imi6E6CeiPwLwJIDzhBDaDhkc7cPEgmnUDwBtM29VNIo79BLQR+VEwbSEgS6s0e9+OMswmEYHObGb3HNoZdWIrYG7EOIjmpP8johOKWj9pwB4U3GM/sL/LxPRvwCYD8CgPRLDOAgjUIKUG7bt8LJzqDRqQG4OClri2ARTB6VuZWBvVzVQt8swzO2wVgFBJjCnw5u7ZiWeqDb/bgCfKvz9KQAPuncgoolENL7w9xQASwA8H/G8NU3Xrj4sWf8k5qx+GEvWP4muXX3VHlLyCVtL3zRpK5O1zBWqc8js+ZSKV7vPZIErfqDOSo7DZm5Pbrpx26asE+8GO7Zfk3smUUQV/usBnE9ELwA4v/AaRLSQiO4o7PMnAHqIaA+Ap2DZ/BtW+NuloPsGchAA+gZyuGnrPp4A/Agbq60Kv6SU12lqZ+GqzmFHo1yxGRg6qs8ezk7yzwFIZUpbL9qJVbLj+tnMTVtaFic3DS0zrGvWla4ogawyGX5N7plEEcnhK4R4G8B5ku09AD5b+PsXAM6Mcp56YtO2AyXlewEgNzyKTdsOcGkIHWFjtVURKSLvtZ+rGpC7z/HEOst5rCKTHWue7jQhTfqj0qzX9/9lqcBU2eP9auCbFkUzNYGdt0bTjN1FdlJ5WkcyZYczfCsMl4IOSdhYbaUJRbLd9Bx+E44deumMWz9vDdD7K33Wq0o4N52kF/w+jemNx+08l4mJyTnJMTUHC/8Koyr5zKWgfQgbqx3kc6b7ahOKZsoFtYnZKujqxq9xuvtzJgJ9aND6X5snocnAZWoGFv4VhktBh8DZjF1Wg0aHadmAIPuet8YShG5SmeAlmp3bg65u/Mw4WVfUtUktImdjdfe9uOx7lolHl4HL1Axc0rnCcCnogLjt2bIaNH4ECTt072s7UmXhpc5wST+7vCps0img/RK03Pg5bk+8OxapZF8bMJb45q42mspYzua1rWPXWq3WiJwnUHa4pDOTbKrZszXOUsAb5iiEv8thair0fvYVq8SCH6ZN2bMTrQgmpyO7WmWPG6wEc9xwA3emPlBmmWqyceMizoknzuvY21mIxjH87V7xg/DN3SvZGL04GSlWNNyk3Qiu58/UB9WsyBhnKeA4r+OJdQhUdmHr56yVhy4hrtplj01yEBq8BHPcsPBnkk2UiowmiU864hTYcVaWDCME7ZaSqntQ7bLHJjkIDV6COW5Y+DPJJki0jpOw5SCcxCWwo0QryQgrBHUZ0dUue+w3oXEJ5thh4c8kH5MmH25M4ur9VgZhJx4neztL6/aLUStM9NQLrLGEWZVIQzYLJRZUSW02KiEb5Fqjrqhk+OVOsLM3djjUk6lP/GzYpiURolanfPRGbymI0aHSSB3VuVXo6tP7lWzWVS41uVbT+xYUWZirOxSViRWO9mHqE7/olUpFt6xtMd83rjo5ezvlJZszWeCsa7yNVmwh2zLTP56+nPetJNrHJfg51NMYjvZhGpsovW6rRe5wPCaUuR3WJFIsD+1TuTRIO8ty3jfbvNcyEx6N36SCKxMIFv5MfRK1121QVHbwYs9gQ+IUcDJfiZ+Q9hOylYgKSuLEXIew8GfqF52jOM7oFl1k0cUbrLIJpugEXByOVhMhrRtDJaKCqh122iCw8GcakzgieWx0kUVzO6yeuKYrAJWAiyN0FQhW3E1GnPctyBg51DN2ONqHaVzi6jPrZ6aY2+HfOhHQCzi/CcYUv+JuJkK23P15ddFMTGyw8GeYqLTMUETAODRorb2a/AVcWezgNFZVNPdOsoQsN4AvOyz8GSYqp14A9NwJrQatnCAMQyRNJhgT3HH6ucOFxvGbWdg2GGzzZ5go7O204uZLQhPJiqeP08Eclx3cJPOZaQhY+DOMmyBRNdKCZMKKp3cS1VEal6OVwyiZAmz2YRgnQcsXKIWpxEQT1Y4dhx08LvMRU/Ow5s8wToKaRZRCk+LJ1o0bDqNkCkQS/kR0FRHtJ6I8ESlrSRDRRUR0gIheJKLVUc7JMGUlqFnkvDWwwiXdiGTa0SsRp8/UBFHNPs8BuALAP6p2IKI0gO8BOB9AL4BniahbCPF8xHMzTPwENYvM7bA6ZclIqh2dwygZRNT8hRC/EUIc8NltMYAXhRAvCyGGANwL4LIo52WYshHGLKKqoc92dCbBVMLm3wbAqUr1FrYxTPIIYxaphB29HA1UmIbG1+xDRD8HME3y1leFEA8anENhEJWeawWAFQAwa9Ysg0MzTBkIahYpdzmCcjVQYRoaX+EvhPhIxHP0AnCui2cA6FecazOAzYDVzCXieRmmcpTTjv7ojfHU9WEYB5Uw+zwL4FQimkNETQCuBtBdgfMyjJ5aMKXs7VQXhEuqQ5mpCaKGel5ORL0APgDgYSLaVtg+nYgeAQAhxAiALwLYBuA3ADqFEPujDZthIhJXieRyE6axCsMYECnUUwjxAIAHJNv7AXzU8foRAI9EORfDxEpcJZLLjV9jFYYJCWf4Mo1JrdS4UWn32Uljk1QtmK+YxMHCn2lMaqVVoCqM9OIN1t+1Yr5iEgcLf6YxqZUaN355B1yimQkJV/VkGpNaahWoCyMNY77a21kb182UFRb+TONSDzVugtYi4oQxpgCbfRimlglqvmIzEVOAhT/D1DJBaxHVSpQTU3bY7MMwtU4Q8xV38mIKsPBnmHrG7dw99QKr4bzT9JPEKCem7LDZh2HqFVkOwJ4fA2ddw528GNb8GaZuUTl3X3gMWPlcdcbEJAbW/BmmXmHnLqOBhT/D1Cu1UsKCqQos/BmmXsbUcZoAAAMLSURBVKmVEhZMVWDhzzD1Sph+xEzDwA5fhqln6qGEBVMWWPNnGIZpQFj4MwzDNCAs/BmGYRoQFv4MwzANCAt/hmGYBoSFP8MwTAPCwp9hGKYBISFEtccghYjeAvBqtcdRZaYAOFTtQSQEvhcWfB/G4Hth4b4P7xVCTPX7UGKFPwMQUY8QYmG1x5EE+F5Y8H0Yg++FRdj7wGYfhmGYBoSFP8MwTAPCwj/ZbK72ABIE3wsLvg9j8L2wCHUf2ObPMAzTgLDmzzAM04Cw8E84RHQVEe0nojwRNVxkAxFdREQHiOhFIlpd7fFUCyK6k4jeJKKGbr5LRDOJ6Cki+k3hd/E31R5TtSCiCUT0KyLaU7gXtwT5PAv/5PMcgCsAPF3tgVQaIkoD+B6AiwGcDuATRHR6dUdVNe4CcFG1B5EARgD8rRDiTwCcDeALDfydOAHgXCHEWQDmAbiIiM42/TAL/4QjhPiNEOJAtcdRJRYDeFEI8bIQYgjAvQAuq/KYqoIQ4mkAh6s9jmojhHhdCPHrwt/vAvgNgLbqjqo6CIujhZeZwj9jJy4LfybJtAE46Hjdiwb9oTNeiGg2gPkAflndkVQPIkoT0W4AbwJ4XAhhfC+4jWMCIKKfA5gmeeurQogHKz2eBEGSbRyexoCITgZwP4AvCyF+X+3xVAshxCiAeUTUCuABIvpTIYSRX4iFfwIQQnyk2mNIKL0AZjpezwDQX6WxMAmBiDKwBP89Qoit1R5PEhBCDBDRv8DyCxkJfzb7MEnmWQCnEtEcImoCcDWA7iqPiakiREQAfgjgN0KIf6j2eKoJEU0taPwgoiyAjwD4rennWfgnHCK6nIh6AXwAwMNEtK3aY6oUQogRAF8EsA2WY69TCLG/uqOqDkT0EwD/DuA0Iuolos9Ue0xVYgmAvwBwLhHtLvz7aLUHVSVOAfAUEe2FpSg9LoT4memHOcOXYRimAWHNn2EYpgFh4c8wDNOAsPBnGIZpQFj4MwzDNCAs/BmGYRoQFv4MwzANCAt/hmGYBoSFP8MwTAPy/wEwani1EnF5GAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(X[y == 0, 0], X[y == 0, 1])\n",
    "plt.scatter(X[y == 1, 0], X[y == 1, 1])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用不同的算法训练模型，比较分类准确率"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 使用逻辑回归进行分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.816"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "log_clf = LogisticRegression(solver=\"liblinear\")\n",
    "log_clf.fit(X_train, y_train)\n",
    "log_clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 使用SVM进行分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.928"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.svm import SVC\n",
    "\n",
    "svm_clf = SVC(gamma=1.0)\n",
    "svm_clf.fit(X_train, y_train)\n",
    "svm_clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 使用决策树进行分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.824"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "dt_clf = DecisionTreeClassifier()\n",
    "dt_clf.fit(X_train, y_train)\n",
    "dt_clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 综合不同算法的预测结果，少数服从多数，给出最后的结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_predict1 = log_clf.predict(X_test)\n",
    "y_predict2 = svm_clf.predict(X_test)\n",
    "y_predict3 = dt_clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_predict = np.array(y_predict1 + y_predict2 + y_predict3 >= 2, dtype='int')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 0, 0, 1, 1, 0, 0, 0, 1, 0])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_predict[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.912"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "accuracy_score(y_test, y_predict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用scikit-learn的Hard VotingClassifier进行分类"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> Hard VotingClassifier：少数服从多数的分类器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import VotingClassifier\n",
    "\n",
    "voting_clf = VotingClassifier(estimators=[\n",
    "    (\"log_clf\", LogisticRegression(solver=\"liblinear\")),\n",
    "    (\"svm_clf\", SVC(gamma=1.0)),\n",
    "    (\"dt_clf\", DecisionTreeClassifier())\n",
    "], voting=\"hard\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VotingClassifier(estimators=[('log_clf', LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='liblinear',\n",
       "          tol=0.0001, verbose=0, warm_start=False)), ('svm...      min_weight_fraction_leaf=0.0, presort=False, random_state=None,\n",
       "            splitter='best'))],\n",
       "         flatten_transform=None, n_jobs=None, voting='hard', weights=None)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "voting_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.912"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "voting_clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "查看训练结果，可以看出，scikit-learn中的Votingclassifer跟我们上面自己实现的结果是一致的"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用scikit-learn的Soft VotingClassifier进行分类"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> Soft VotingClassifier：计算所有算法的预测的概率并且取平均值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "voting_clf2 = VotingClassifier(estimators=[\n",
    "    (\"log_clf\", LogisticRegression(solver=\"liblinear\")),\n",
    "    (\"svm_clf\", SVC(gamma=1.0, probability=True)),\n",
    "    (\"dt_clf\", DecisionTreeClassifier(random_state=666))\n",
    "], voting=\"soft\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VotingClassifier(estimators=[('log_clf', LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='liblinear',\n",
       "          tol=0.0001, verbose=0, warm_start=False)), ('svm...       min_weight_fraction_leaf=0.0, presort=False, random_state=666,\n",
       "            splitter='best'))],\n",
       "         flatten_transform=None, n_jobs=None, voting='soft', weights=None)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "voting_clf2.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.92"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "voting_clf2.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 集成学习：放回取样和不放回取样"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = datasets.make_moons(n_samples=500, noise=0.3, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD8CAYAAACfF6SlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJztnXucHNV153+nWz1SD2Rn9HKERpKlZFkcCEKyHsErrW3AvIwRA5gRJk7sxLbwYptYzgrE2iuE4th67Ab8Yh0Z8wEn2GgwYhgMfAQGEiI52IysF8LW8gpoZsAgxMhI09I8+u4f1dVTXXXvrVuP7q7uPt/PRx9NV1dX3arqPvfc8yQhBBiGYZjGIlXtATAMwzCVh4U/wzBMA8LCn2EYpgFh4c8wDNOAsPBnGIZpQFj4MwzDNCAs/BmGYRoQFv4MwzANCAt/hmGYBmRctQegYsqUKWL27NnVHgbDMExNsXPnzkNCiKl++yVW+M+ePRs9PT3VHgbDMExNQUSvmuzHZh+GYZgGhIU/wzBMA8LCn2EYpgFh4c8wDNOAsPBnGIZpQFj4MwzDNCCJDfVk4qFrVx82bTuA/oEcprdmserC09A+v63aw2IYpsqw8K9junb14aat+5AbHgUA9A3kcNPWfQDAEwDDNDhs9qljNm07UBT8NrnhUWzadqBKI2IYJimw8K9j+gdygbYzDNM4sPCvY6a3ZgNtZximcWDhX8esuvA0ZDPpkm3ZTBqrLjytSiNiGCYpsMO3jrGduhztwzCMGxb+dU77/DYW9gzDeGDhzzAB4LwJpl5g4c8whnDeBFNPsMOXYQzhvAmmnmDNn2E0OM08QrEP500wtQgLfyaRJMG27jbzqOC8CaYWYeHPJI64bOtRJxCZmccN500wtQrb/JnEEYdt3Z5A+grmGnsC6drVZ3wMnTmHALS1ZvHNK85kZy9Tk7DmzySOOGoS6SYQU2E9vTWLPsk521qz2LH6XOOxMEwSYc2fSRxx1CSKYwLh8hhMPROL8CeiO4noTSJ6TvH+h4noCBHtLvxbE8d5mfokDqEbxwTSPr8NVy5oQ5oIAJAmwpULOGOaqQ/i0vzvAnCRzz7/JoSYV/i3LqbzMnVI+/w2fPOKM9HWmg1tW49jAuna1Yf7d/ZhVFhBnqNC4P6dfYH8BgyTVGKx+Qshniai2XEci2GA6DWJ4ihqF4ffgGGSSiUdvh8goj0A+gH8DyHEfvcORLQCwAoAmDVrVgWHxtQjfhOIXygoN8Nh6plKCf9fA3ivEOIoEX0UQBeAU907CSE2A9gMAAsXLlQlVDI1ShISt5xj8cslUEX7cFIXUw9UJNpHCPF7IcTRwt+PAMgQ0ZRKnJtJBnHE3Xft6sOS9U9izuqHsWT9k5Fs7ya5BBztw9QzFdH8iWgagN8JIQQRLYY16bxdiXMz4YhbS49qP+/a1YdVP92D4VFrQdg3kMOqn+4BEK6ipolJh5vhMPVMLMKfiH4C4MMAphBRL4CbAWQAQAjxfQAfB/DfiWgEQA7A1UIINusklHKULo5qP7/lof1FwW8zPCpwy0P7Q43J1KTDzXCYeiWuaJ9P+Lz/XQDfjeNcTPkpR5RLVPv5O4PDgbb7serC0zxF29ikwzQSnOHLeChHlEvS7Odx5BLUInH6TZjahmv7MB7KEeUS1X7ems1gIOfV8luzmUhjqndh74Q7kTFOWPgzHsplEokibNcuOwOr7tuD4fyY3T+TIqxddkakMTUSnLTGOGHhX0NUKk4+iVEufmOKem+SlINQLjhpjXFCSQ26Wbhwoejp6an2MBKDrKtUNpOuazu1qUCW3RsCIGDZ8v0EeaPc2yXrn+QS1Q0AEe0UQiz02481/zITl0bZaEt2P/u0876miIrF12zsVyZ27Ua5txzhxDhh4V9G4nSw1cqSvRKTHYCS++oW/G78BHmt3NuoJNGcx1QPFv5lJE6NshbqzFRqsjPprWt6PKA27m1cNFqEE6OG4/zLSJwaZdLi5AFvzPja7v2Re+/atDbLQzhbmzOh7p9OkCfx3jJMuWHNv4zEqVGWc8kexlQj0/JVhBHWKkuOEOr7mi7Y/m1nr42fIGdzCNOIsPAvI3E72MqxZA9rqgliegkz2R2RJHTZ29cuO0MbnRNmMmNzCNNosPAvI7WgUYb1S5hq82EnO92qye++RhHkjRDvzzAAC/+yk3SN0sQvIROIKuE8sTmD5qZxkYWn36rJfV9t/0OU83L5A6aRYOHf4Pj5JVQC8coFbbh/Z59HON986RmxCMogq6awQts9qR07MdIQ8f4MA3CGb8Pjl92qywpddeFpiTCRhMlclV23jrbWbNWvk2FM4Axfxgg/DVtnFopievGzrQexvasijXR+iSAOa3Kcg01BTL3Awp/R+iVMw1WDmF5MSje431+5ZTd6Xj2Mr7ef6TmWO7RTNUYnpg5r2bHZFMTUA5zkxZTgTtw6531TjRKgTBqim+4re18AuOeZ1zzNRzZtOyAV/ARoo4x0E0OaqNjgRWUUrbfSD0zjwcK/jojapcnWuPsGchCwNO77d/bhygVtvh2vgmQzq/a1Vxiq9wXgmUxUJh8/T5ZuYsgLgVfWX4Idq8/FRE2mMcPUMmz2qRPiCFNUaeRP/fYt35K/QbKZVfsCwNe69qG1OaPszeueGNKSip42uutvn9+Gtd37pd3BnGPWZRozTC3Dmn+dEMTsoiKM49QmSH2cVReeBlIc555nXsNxjSPWPZnoKnrmhkfxt517iisg98roY2ed4jtmXaYxw9QyLPzrhKhF5GzHqQyT8gxBGqK3z29TmmUEgNxwXnkeWzDbgtyPUSFw09Z9+FrXPqVJy9kHeEKm9CehuvZ6rPjJNBYs/OuEqEIqrOPUJkjXrXm3PGY0Jjet2UxJNJCumJyT3PAofvLLg9KV0cN7X8eJkbHJ5p3BYdy0dV9xtcAVP5l6JRbhT0R3EtGbRPSc4n0iom8T0YtEtJeI3h/HeZkxogopnZPVtMKnU6t2ClD3fjI7u5OJzRnptdjN2sPU81eZh94ZHNaay4KsaJiA7O0Ebv1TYG2r9f/ezuoco0GJy+F7F4DvAviR4v2LAZxa+PdnAP5v4X8mJqIWkdOVSe7a1Re4wqcsFj6I0L5yQRue+u1bgRLP7PH6dfYywXmOpNdnqkn2dgJd1wH5giJw5KD1GgDmdpgf46HrgeHc2DEeuj7YMRqYWIS/EOJpIpqt2eUyAD8SVi2JZ4iolYhOEUK8Hsf5GYsoQkpWSA0Ys5nbx5dh6m8w9T+8MziM+3f2KTXslmxGunpozWak5Z5VZDNpjB+X8o34ccOVP2Pg0RvHBL9Nftjabiq4n1g3JvhthnPWdhb+vlQq1LMNwEHH697CNhb+CcEWXn/bucejOdtRMzamFT7dAlQX4ulGl0VLCs80UekKSHcuuzYRAG31UDdxFZGLMmHUxeSTOxxsu4wjvcG2MyVUyuEr+7l61uZEtIKIeoio56233qrAsBgn7fPbkFeYTEaFwKqf7sGq+/Z4bPumWcAyv4QO1UphQJED8M7gcDECaMfqc9Gm0N7tgm/2SimITT9MSK3KJ/K1rn2Bk/JM/SsNQcuMYNuZEiql+fcCmOl4PQNAv3snIcRmAJsBq6pnZYZWHmpVO9Np58Oj3kdiJ4F984ozfa/X7Zfwe8Aq04vK7AOUauKmndT8zGXOZxmm3INqwrjnmdeKx4vSQa0maw1lJ8m1/Owk82Oct6bU5g8Amay1nfGlUsK/G8AXieheWI7eI/Vs74+abVutiaNrVx8Gh0YCf05W4VOFc795tzymFOI604vK7GNjC0M7KznKvTQt/RymiFyQgnH2dyJIIl6iFZCLNwAPfgEYHRrblm6ytpti2/WfWGeZelpmWIKf7f1GxCL8iegnAD4MYAoR9QK4GUAGAIQQ3wfwCICPAngRwCCAv4rjvEklinYW1abcN5ArRry0BfjBB61v7yRswtPaZWdg1X17MJwvFYMTmzPapjAqs48TWxhGjdQxiVDyC6kN4utQCXG/Z5MiwpzVDxeFPIDI5T7KSlyCe24HC/uQxBXt8wmf9wWAL8RxrlogSrZtmInDLRxsh22QH7yJkMukCRAoEdZRG9Lb5w6inZoI07gycP2emckEu+rC06STnAzZuE2ejfuZjx+XSr55iAV3VeHCbmUgSJEzoDw2ZRvTH7yfkLO1cft8cZkSVJq5zmShCku1iTMDVzfREOApeKca9y0P7VcWq/Mbd9C8htzwqPLecClqBXs7G858xMLfkCD2U1NHo33cctmUTd+3j6/Tpo8X6u0ENaOEsTv7mb7cK4aWbAZEljmotTkDIYCVW3Zj07YDkSenVReehpVbdhs1i9GNW2eqosKxVGNVPRu7tWQQuCaRhAZNFms44V8OYeQmiDmjEjZltz3Yro/jHN8575vqacjuxL2CMLmPYfwXXbv6lLkGzvPbk4BzHK3NGRw9PlI0r8Rh526f34aeVw+XROYAwRva6AS4rs+w7cdxdxSzz69yAk9szuD4cN44f6GhiSNZrAZXDg3VwN2vWbmKMA3CTZmz+mGlqcdPI9QJBxXZTBpXLmjzCPpsJo33z2rBL146rB3PrcvnSevg28d1lmQYHBqRmjpasxnsvvkC6fXoVkEE4JX1lxjvb+P3nLp29ZWYZexM4aA9hVXP0r5vQb57smuzn3GbaxJXHReI10RXt6xthfzXQ8DaAf/Pu1cOgBVyeum3qzIBcAN3CWGjcKKWS9YRRiMEvMJBYEw4pAhQ+RbtCpcyzfqZl9/RTiAt2YxS2OaGR/HPz7xWfK1biQzkhqX1gm55aL9WkLtNFqa1gnRj6drVh1U/3VOSwzCQG8aq+6yMZvdKQ3UMVVVUe9xBnduqVpbu74XfcVnYG9AywzL1yLabUKNlJhpK+IcV4kEduEEI4h9wohIOJzWlkRfQCkVV4TNdQbRsJg0i/XGD4J5wv9a1T+sQld0T08k3rUkM2LTtgDR5bTgvQoXmunGOO4i/JMh3NYzTnHEQNVmsRstMNFQ9/7A178tZ0z1syWCVcDg2pI70sFEJQ932b15xplF8vZNlqe3Y3nQ9Xh5/DbY3XY9lqe3F95zj79rVh3scqwbV+d33xHTyHRVCWf5AN4GEDc114m4OY0rU/gxcBiIAczssE03LTABk/e9nsnGWkibFM054mYmG0vzDatlRyyX7IdPc/LS2IIlDTnQ2f9V2W/D6FUtzsiy1Heszd6CZrAzOGXQI6zN3AMNAd35piRDTmUwA4P90nCW9134hn05Ujl/dfTQRtH4ThN0cRnZuGSZOXhPqpgxEpQiSc+C28QvJ968Gykw0lOYfpTFH+/w27Fh9Ll5Zf0mxKFi5MNHawqw6WrMZTMikcM8zr2FCJoXWbKbkPny9/UzcMmc/doy3tPUd46/HLXP2l8TXywqzpSQLhhvGdRYFv00zDeGGcZ2e8esEqN29S4b7ebZmM1YimgRV8bVVF56m/MyxEyO+mrLJBGHaS9ndocz24wDBm8iU00/V8Mhs/ABAaRivHBJAQ2n+QG005jDR2trnt0mjbgBINUa3Vv/O4DCymTRuXT6veMxnu/8Rl766HtmC0G7DIVz66no8230yFi27VrkCWrllt2cM0+mQ9Nqm09vF8Re3KbRvAordu1S4n2fXrj58WTIeQG0vByBNwhrISbR2V0jfbad/CX/57Ht9Vx9hTUgyJ68JJn4q9gmERGXLF3mz6KCE0FCaf61gqrV97KxTPLWys5k0/vzsWZ7VzVO/fcu3FHHbrzcWBX/xeDSEmb/eVHwtWwHJtN9+MUV+DWIyJjZnig3Y56x+GMdOjHi0bwLw52fPCiyM2ue3KUs5q7T09vlt2LXmAunnSu6Rvdw/chCAAI4cxKJ9N+OWOfu1TmXduZ3Eqa37+anYJxCBOiklzcI/gZg4+7p29eH+nX0lGj7Ban/49fYzPQLaT7B07erDNCHX1t+j2G4jEzS34WqcoPEl2wZFEzaOdOD0U/6gRPAM5IYBYSUm2RPWrcvn4evtVqx60D6tKvPU4JDejOMrfBUhfUtevd03UsrETBfVyevEz8QZtC+Bc7I27T0QiST35j1vjWXTd1IDNn43DWf2KQdxL59NHNMqE8FTv5U3wfEzA2zadgALxRTMkJhr3qQpmKYZr8wctPTC6/DNh8fhs0P/jOn0NvrFZGwc6UB3fina/+MhPJ7egunjD6FfTClub24ah11rXMlfezsx8uCXMG70uPX6yEHrNaC0qdrjcZvF/JyvvqYSxXL/FLzt2ZYmQl6IQN+HsAEJKnQmziCrjKglyj34ZcMmvdxCnZSSZuEfkdh/GDCLLgpqIpAJFiqM185g3pjqKInQASxt/eCCVVrhb4/Zfb0rtyzGXVhcsm1Zaju+MU4eBfTQwFLPcQcfXYNmW/AXGDd63Nqu+bHZ0Ulun4gu4sVX+CqSgfrFZM+2vBAl2cgq3IqDnSVtl+Z2auNx2uOD5K7EGjlkIthrIWmqDiqSstknImHa+pngF10U1ETgNAMApU5hO6ywO78Uq4c/i978FOQFoTc/BX9Hn8eiZdeGugbZWHRRQLL9J+TekB5btb3I3k5sGfycb46BE99oMMlyP4fx2DjiFQImppquXX2etphbfnWw2BbTXaY5TlNLkNyVWCOHdILdpkaTpmoN1vwjoooRDxODH4QwJgJbO5fVKrIngu78UnQPWRp4JkXYdNVZsY7RjvZxM53exjnvm+rZ3p+fjBkprymqPz8ZSvdaQbuckbKuUZdj4EYbDSZZ7j/3x1/C48++F8gHN9Ws7d7vqfE/nBeeInKApVB82adSaRDzY5DclVgz3E0Ee9RyC4wRLPwjIqunbm+Piu7HHCXxzFhji3gJsjEep2loznk7ePaLyVJ/xR1Nn8QNw7d7TFF3NH0Sa6G4R//i1S7t1cXj4kPRMrPndqBrdIl1zt/lMP1EFlcumFpS0O6c903Fpm0HsHLLbu1zUbWw1CW8qcyKYcyPpmHPsfoiTAQ79+atCA1V1bMczF79sPK925bPk9acNxHUqpoxfi0OTVBVKZURR+XSEvZ2YvD+L3iE+erhz+Kh/FKPnbxrVx+2P3A7vox7i47j23A1ll5+HQBIhdLz6atBEhGaB6H7sv36e+dwRg5mp2Hj8HLcfXSxsj2ifU7bPBSkcqzuu+OH+7mUs/IsEGNQg2kFzFopkZzAcXJVzwrRplgST2wurYDp1PJMtDJVzZig5QJkBCmLEHtG6NwObOzeL40CksXZW9d4HZZvO88jeJasf1Lqb+kjuakolZ3oL/gdgqk59zpuELfjcGoI3QNLcdPWfZiQ0bdHDOIcndickRazMynO1zeQK6mMWu6M3tiSI00jZWyHqi1ct66w/k+AcC2S9KgkH1j4R0S1JBY+P16/aAndjzZqjRaZOUZVe78cnZ/mXbIC52/9AHJDZmYEleBR3aONIx3435nNaKKR0jdOvGv9YFU/TIkz0jYXdQ8tNWqPGEQI33zpGZ5y0pk04e8vH6vFr1uhOZWAclaeDY1KKzaNlEm6cK2FqCQNHO0TEVWEyBGFPdeJTsD7/WijanTuaKJL5sqzhcvR+SlKjSUnqnvUnV+Kd8UE7xv54WJUiSxpSSickSontWwsqjG1Nmc852uf34ZNHz+r5D5s+vhZxclux+pzcdvyedKENaA0qkwXvVPxBC1Amg2Nh64PlqxlEhlUTWo8KqnuNP9q1CuRaaYmFTBlgsK0O1ecGp0uW7hcdeLjMCPozFcT6aj8Q0d6lc7RRanJaJMkuREEtjddj40jHXh6/Dk4MaJujygbUyZNOHp8bGWl6kksw37Pr16RKgAAQOx5KEbEoRUnXbjWeFRSXWn+SapXoioxYCPTqmVVHU0/K8NU4wuSLVzVe+xK+W9P7yjJXXCiqi2ElhlKu/yG4Q4MiibPR4iAGalD2JC5Az+Y/4p21SJb1ZzUNM4T0hkkF8S0XpEsN6RceSi+xCG4k15Dp8bLPMSi+RPRRQC+BSAN4A4hxHrX+58GsAmALSG+K4S4I45zO0lSDXO3JmYS7aNy8k5szqC5aVygUMIgoX9B7NSVusfu1cVtp7+ARftu9th/2y/9NtpXd3iud+NIBzZk7igtVFf4Yfb/WH693fmlwLCViNZGh+CO1s3SEBa99J2SCqcy3Nr8HEVUj5/pznkPWgrlqp3+ARMloGqlnePQipMe8lnjZR4iC38iSgP4HoDzAfQCeJaIuoUQz7t23SKE+GLU8+lIWg3zoKYN1TgHBoeLNW9MhXoQIR3EWViJeyy7xuk7NwKkNiO4J9ud/+l8PHf6bCx66TulYZs/PgkpRW4GMJbk9vL4a+RpDiFMDmGcse57MJAbRiZFmNicMQ4XDnvuWIhDcNeCcK3hMg9xaP6LAbwohHgZAIjoXgCXAXAL/7KTyIiHAKjGnyLCnNUPY3prFgODQ0ZCXSWM7Vo+TsFxzvumerJKVVplJe6xbOI6BYrKog5hXDLZ7u0EnhgT/GuOXYmfDll1hnQVOG36FUXuwpgcbD/A+aP/apWxoEN4HVPQf/oNAEpj8J0+HzfDeSEtfqfzwaj8Iu8cO1H8TpXFLxaX4A4jXBMYe59E4rD5twFwru96C9vcXElEe4nop0Q0M4bzeihnr91KoPITjApRtK8fG9KHGtrohLHTTh/U2VuJeyybuHQ2fA+uSJPm3OtYR5tL6vsA+izsjSNe+38O40OZHNrnt+FHi17FhqYfYkbqEFIEtNEhy4zliH5x+3xkuO+Nnw+mfX4brlzQ5lnFDA7ny++zmdsBrHzOanCy8rnKCOAwUUZJLh9dRuIQ/rJfkFu1egjAbCHEXAA/B3C39EBEK4ioh4h63npLXppYR1whhNXCPf4gJSLcwt7P4WyvFoKWhja+xxF+ULKJa+NIhyV8najMCJp4fSd5IXDb8nnSNo6yInerhz4TWoAteuk7yOJE6UZX2KJfM3jAe29MHLpP/fYtbcmI2B3A1RSmQcND4whJrVHiMPv0AnBq8jMA9Dt3EEI4A6V/AGCD7EBCiM0ANgNWeYcwg4ktE7FKOMevchTKcGveThu4SpPU2el17/ne44jJOTJTxePpD+Ev3j9mw9cu5xV2+TY6hJfHX1PsIbDzP51fvI6VnbvhtgY5i9wBUEbcGGEQ/eLnN5GtsEyerYk/JjafTbUTs4JGGdV4olYU4hD+zwI4lYjmwIrmuRrANc4diOgUIYRdzWsZgN/EcN6KU+4cAvfxW7IZZfEvJ63ZDACrvkv/QA6tzRkIARzJWY5BVRkBXWZviqikfEAgIv6gVDHri+ZfBMCgvLQi0oTIWqbOICts87nTZwOwQiJ7Xj0sraZZ/Cy8E6wT3++GQfSLyp8CWBOP+5hdu/qUuSDOFYLuuLL9tfjZ06stTE2ijJzXoHriScklKCORhb8QYoSIvghgG6xQzzuFEPuJaB2AHiFEN4DriWgZgBEAhwF8Oup5K02Q0Mkwk4Ts+Jk0IZMiT4y4k0yKMDyaL0kCcgrzvoEcMimShgme876p2PIryQ8Flp8hdDJQDDHekVZwskgTF3bYJnCt1O/hRkB9H4y+GwbRL6pSISrT5aZtB6Rjdk9UMoe+E2OfjYlWX+3ELL/7LCssJyMpuQRlJJY4fyHEIwAecW1b4/j7JgA3xXGuamEaOuknCFQTg+z4w6NCGuNvlw9uyWZwbGhE6QQuHicv0JrN4KTx40rOu2nbAe3EEjp+v9qZj+5IEx/tzsTW7jT5uJ/h4NCI/3ejMKbBR9dgQu4N9Ocn4w7xScwbXYL2wmfsFchPfnkQo0IgTaR0vANqU41zolJNbM2ZFHLD+WArWBOtPmnP3r06kV2DmyTlEpSRuivvUC5M49t1k0TPq4fxz8+8VtzeN5DDqvv2aI/vjPF3s2T9k0ZmIcAyAe2+ufQ4KxUlA5yEsgVXOznHbZoYOgbkDnv3KwikILZ22eSuwhOZM7oENx29bez7MQRkXYrB/Tv7iqGoo8Jq7AJgrJm9A5U5xzlRKRMHTxqP54OWejbR6iv17HXmJ114qGxiKkINFRpaV+Udyolp20RdfL1T8NsM5wXWdu8P3JbRPqYpsuOY2HntfYyLg9k/yuEcQIVoo5aZ3nrt5UIWvTF0FEhlSvdzCCTdfXBHM5msEmyCRuaoIq/ueeY16f02CbuNMylvMCvv5FyyfW6H9axbZsISpmV49mEjdPZ2QtmhqGVmZUNSEwALf0NM49tVgkQXtjmQGw4cP287+0xQHccvHNS2HRvX8yn5UQIQo2NCNsgPKkqooGxZPzpkVfS0yU4qEUiqe3/b8nme/smmQlN2z/0Esc6MIwvFNAm7DaNUqNg4vNyT+zAomrBxeHnpjuWO7w9b7fOJdZCbAKkhzDxuWPgbYhrfrhIkJlmlQXIUVM4+wLLntmYzvsexr2lic8bzHgH487NnBSsOFkcJ3qhJOtplfYGR0jHaiVD2BK2ztTuF5rLUdmxvuh4vj78Gv5hwPT598q+091wlcFsK0Vo6gaxrOu8u5mbTtasPg0Mjns+ETcq7++hib+7D8Gdx99HFgY8VibBOZeX7omG0fSds8w+ASQSKKkzRr8Tzpm0HPD9eHToN9Pm/u9joGPZ4dY5o3bk821WCN0ikR9BQQdPoDdfxBrZ+BU2PrkFz7g0MZqfh+LErMSr+KwDL1n7/zj4sfO8k6eRul2pYn7mj2I5yOg5hLf0j1l5zBjB3rBWl8762NmeQApB3DefY0Ai6dvVh1YWnYeWW3b6hmyao2oC2ZjNYuyxcG9DprVl0D5TmPgAR8x+cmJZlCOtUVn6uLAUHEg8L/zKgmiTcXZucBLXBmjj7goScysZsf96op0DRnirZO0ikRxxJOga0iHdBuXcBoFj+YSiVtyp7Qh3pZL8++8EvohlDpQd1TVJuASzLpwCsqC578pflG4TR1FW+iZPGjwsdQhtrI3c3JmGkxcnhIDzfNROnchhndB3XCWKzT4WwuzalFIb6oJqdn48gat19vzoznh99XPbUoDXcQ8aPu10wzTSEm8f9qGSbztQyzaDQ3KZtB3D+6L8WTUPbm6731Bdyn+vr7Wfi1uXzjMx/Oido63eZAAAgAElEQVR8OaqvlrV8ip/J0O1PgkDReWvqVA7qjK7z0g+s+VcQ+0cSh/bkNi/ZWb0rt+zGpm0HcOyEQey5Bl1Ui51tCoxlFb80oVehSQS0pwbVzlRLeQ1CeIU/AEyio1iW2l7U/rUTsuK8b2AKnilkRi/4/eMlpqEZdAjrM3cAwyiew8bdlCVMUqCqp++y1PZiNdE3aSqw91ho7bVs5VP8VnzSFZ6wIsqCaONBqoRWO1u5zLDmX2Hi1J5sZ9+ty+fh+HAeA7nhopaviv831fxU+9kRQGu79+PLW3YXVxb9+cnyAwW1p5pqZ7aTt2gCcI9STV7xPhGKxd98J2RJF6dB0YRvDF2Fm7buw9e69uGGcZ1FwW8jKzDnPpdJWK2fE95eGS5Lbcf6zB3FaqLT8BZGHvxS8rTX7ET9dtXkIEbLp41XO1u5zLDmXwXi1p6ixJ7r9pOZfFqbM2O16ZssbbJfTMET+Xm4ip4uFXZhk3v8tDOPk9c2AQhrsjj1AmDPj6W+gEHRhPtGP4i/TP9cqv1Pp7eldXSkYwTwxtb/ifeIQ+gXk7FxpMPS6POj+MkvD2Jdk9w01JayziHzxcg0+pVbdqPn1cMliV5+Zh37eIsf/ILHNzFu9DgGH12D5kprrzr7+cgJ+WdOHLUmeV3xjXJp436O5Rr3B7Dwr2F0jT9kBDEvqZx7QsAT6TKDDuEqehr3jX4Q56V2Y3rqbRzPTkPzxWVaHqtMAC0zrbhyAJh19phzkNKAGMVg9hSrm9eJxbhs3C/Rinc9h061zsCOla7MV9WPfG4HPvDjk6RiaVQIZUMYapGco4Au0csZfaRrquN09L80/pB0ITQh94b0/MYEFXx+Dt3hY/LP5YfMzHrl0MZ1JshqVy+NARIG8efVYOHChaKnp6faw0gsqlA+J+66QKpev6qIINl7K7fsxr81XY8ZKa9QGxEpfGX48+jOL9UWJANgJjxU+6xthdK5vHZAd9tKjy37YbtNTLL90k1A08lA7h28gSn4xtBVHht+mgiX0L+VTJKA1RAme8V3lQJizuqHlTpuW2sWOwolGWTPP5tJ48oFbbh/Z19x+3bFs+rNT8GMdS8pzuSD6t6ddQ3wwmPyZ1o00bmwJ+y1LeHG4j6ObKxRtHPV5/2up4oQ0U4hxEK//Vjzr1H8TD3ZTBo3X6qP5/ZzGsrMU5u2HcD0nNycMY7yYw7N4aVq57JpWJ9qnziKh5m2GVRlDBdqBU3DW9jgcuKOCeEPFRvCT6e38Tomo3/BDVjkOEeQMt5OU48un8T5vdg40uGZgAZFEzaOdOChsG0cVY7QnjtRnJTdz9TPfp6dJK+/ZILKvBiHdq4yQdaBP4A1/xpltqbRi5HNGlakjipXYIei6FfXrj4s6vog2mT9bQv05qdg6dC3QQBeWX+JdwcTrUm3j2o5Xo76QcpVRilvYCo+cPxbJcLUL89Cpr27S2870T0XG9nKYSza5+1S30QB31WamyBauv1MVc/T3ufUC4Cdd1kOXD+yk4Cmk/y1+XJq56z5M5XGFigq0kQeAaESQmFiwdvnt+HZgzdg0q//l7ctYYE2OoRfj1+BVjoG3Cr5cZpoTbp94moOboJhKOk0HJJPdBpUZbxPakpjcGhUmejlfp7OMt8pIk8pEXdHMjeBS3cXfChG2M9R12PhyEHLQZ+ZYFVg1ZHJAhdv0MfmV6JRS7Ur18YAC/8awsTO7/7h60w7OqehjkXLrgVmTwQe+LxUCBABk3DUeiFbapuYbfz2CRKvHQWDxjAAPKGKJg1eVJPs4NAobl0+Tzphy47rrBZrUkNKRqDkL1PBD5Q+LwB49Ea5ecckS9te9c3tsIS881jZScAZlyujvKRjCoLM9n/ptznah6kMQZuOqD5ja3qR0vXtL7mJYHSH4ploTUnRrNyrjOxE4PgRrwA88a4lIAr7mzT/0U2+qnDgIGG9QQiUYd4yUzox5wVKM9hlz2skwCRjk50E3PjK2Ou9ncCDX7B8Lza5w0DPDw0OFqKCp8x3sPVz1rh0q5CEw0leNUSYBt86006khDNn3X4TnEttk0SuStSFd6BNrHKWKL7xFWCCxOadHy6pXmpiUtOV6FCNJ4iG7qw66iwtEaR0uBRFgts/jX7Eqvipel5+35fsJM9xi2YeJ0+sKxX8gQhRwVM17tzhmi73wJp/DRG0wbfuM7amFyrhLEwlTfdSW2e2cS+xr9hcVu0qSH9mAEDuHfmBHBOciUlNFbEDILCpzk17ege+Mc5bWmJSpgnzLlkRuMd0Cc6WlINvlDiR12fS+OZlCgVCZ2t3Cnk/U0oUm32YCp6689VwuQcW/jVE0Abfus9EqsQYtJJmEHNNFZJnTPszFzHwWZjed9nku2T9k4FMdW6ymTS+2nQfmkclpSUyW9A8/xZ1CK6pDXtuB5rndpQ4n32jzFT3jdKlqwS/56x1wisqywLhzYZ+Tv8aCu90wmafGiKMmaYslRi1X3aylu/ZSQhlromjIUxAAkc9ScwebsES5b4HNdV98uxZJa+vXNCGySNvSY/RrMrsDVnBUtdMxoPqvl3+/eDF/9JN3u2pDLDwr8dMhVG+h37jdlKp5vQxw5p/jRHGTBN7JUZdU4yoMc5VSJ4JHPVkGGrant6B9vHrgAm9wPgZQHoNAH/hE9VUt2T9k+gXk6WlJZSCqhIVLOMK0ZVFDgVxvgbN+tVFKtVYeKcTFv5McOKKxJH9CKNm74ZI5w9lGnP5LLp29WFTobz19NYsbjv9BSzad3Mo81VUU13/QA4bU/LM3mbVMwoz6Qa51z/7ylgSF6UtDf1j/2B0PVLChvqGNSva56vxYm5OYjH7ENFFRHSAiF4kotWS98cT0ZbC+78kotlxnJepEnFE4qjMDKdeYC3fnaQyZhOL7JhbV1gZqZpG8FFNY7LGOdN3bgxtvoo6numtWXTnl3r67W7MXKd+RkGb6AQxE/3sK1YYph0eK0at1z/7itH1xEpUs2K5m9NXkMjlHYgoDeD/ATgfQC+AZwF8QgjxvGOf6wDMFUJ8noiuBnC5EGK57riNUt4hSKvFuhqHKj0+OwkYOloaypduAi77nrfgmlsDK7b4U1CmEhCyMhkvj79G0bUtQPG5kKiKvvkW2gtSMsOvXINTI75lkjwxjNLAzSHq+UTRvuMoCphwKlneYTGAF4UQLxdOfC+AywA879jnMgBrC3//FMB3iYhEUgsLVYjAIYaa40QR3HGNIxAqc4Is+3N0qNT2rFq6B0k2i3H5LnPQqso5m5qvVM/U5FmrQkhN+hMY3xOdOchtSlFlBAfJFLaJGg0WR1HAclAFc1Icwr8NgPNu9gL4M9U+QogRIjoCYDKgaoTaGAQOMZQQh+COYxyBCdp+0SlsVEt30+PEHE4qc9BuHOnAhqYfltY/MvSLqJ5pz6uHS8o16551KCd/EDu63/NzTrSqWkCU9m7zQ/XsH/i8ZeLzE5xJyRx3UqXeAHHY/GWLW7dGb7IPiGgFEfUQUc9bb8lD1eqJOJps+7Xzq9Q4AqMK+8tOku/v1MwiJfnMiD2cVJap+3j6Q3ju/X8HtMyEAOENTMXfHPsrLHlkirQtoxPVM/3JLw9Gftax4Rf+CFhCbG2rVbBNxoJPBz+vrp2jSYhqhTPHjahCeDMQj/DvBeBMm5sBoF+1DxGNA9ACwLO+F0JsFkIsFEIsnDp1agxDSzaqUMIgdVbiENxxjCMwqh/hxRt8Y+jNl+guncM+TszhpCoH7aJl16Lrw9tw+ui9OPv4t/BgfikW/P5xLOr6IMTaVqUTWvXsVEXbyjpJA2P9kp1jLnl+OoRVqTOVRvF5UBpY+Jlw0T4mz95PcNpO2ys2W6+3rtAGBJSdKvUGiMPs8yyAU4loDoA+AFcDuMa1TzeATwH4dwAfB/Bkku39lXJ+xpF9axKj7nc957xvKu555jVlCeGyoTMz6OyfRpU2yfpxy46jcgzrIlv8YvoNCrHZzdSL4ZeK5b3qmaYl5Zrt/cuGn0nC9p/4PY/8aDx5IOetsYS1X48FP8GZpDaMVfJDRBb+BRv+FwFsA5AGcKcQYj8RrQPQI4ToBvBDAP9ERC/C0vivjnreclFJ56d9vLXd+4vdmyZkgi3G/CYQv+vp2tWH+3f2lfyUCMCVCxQ247gdU5r+uEpKnJOqiJMZ6uP49WZ1jsfdDD6gkHBq5TeM6yxtcA9IE6lUz/SWOfux5NXbcQoOoV9MwcaRDjye/lDJJB274mKS/OV2Fpezjv7cDquiph9+grMSSW2mVMkPEUucvxDiESHEfxFC/LEQ4u8L29YUBD+EEMeFEFcJIf6zEGKxHRmUROKwoQflxEi++Pc7g8O4aes+X7uwjV9MuN/1qBqGP/Vbic8lZAkAJVGOV1y6/8DfTCT77FnXjDkcKW29Brzj6bkzkj3WqZVPV3U/cwlF2TP90aJX0fH6JrTRIaQImJE6hA1NP8SPFr1a0nPZnW8Q5LtkMjbldmf8u8oUFJcm62tqImvS1pGkNoxV8kNwhq+LSjs/44i00UV2+F1PoOuNW1uK43iyEMVTL7Beq6I/9nZa2rwz6WjPj4H9D0hMFyot9qBlJ/ZZ+dx2+guYvnMjTsEh5JFCCnnvTpQq6QUASJ7prV6zShYnsOil7wC4FkDE75JqBRbGJCE1yxkIZFN8zX7Cep6zztYntSUp5LNSzYkccGE3F5V2fpZ7svG7nkDX66ctyRyDOuLSvpxa53lrrB++O8v37mVjY3vg8/JJJ2gDcb+Vyt5OLNp3c1FbH0d5+VQiRv1XPAb3KvR3SbcCMyhi58FeWZU43AsCOQ6nqq0pqyLDAP/VWZjrAoJ/xxMMC38XugYbcdO1qw8pkqaBxjbZ+F1PoOvVlQAIY8IJWlLABGm5aQG88q9jYwucXCR/RgD0QkYyFuWR/ISVwb2yvzPuJi6fOvlXxX2kTWIevVG/AgtjknjhMXhWTc5rjCpE53ZYTdx16JSIMNcVt9mzyrDZx0Wo7MgQ2PZZWfRGnJON3/UEul6dYyqMCaccjq7INltXPfhM1tJiX3hM7VyOawWj29/gXq268DRsf+B2rKPSJi5fE98H9p6BrtElHuf/9gdux2Wpw/JJyR5PGJOE7p7EFWnjd3/9lIig15UkJ3EMsPCXEGcJZFX0haoXa5ooer19F37XY3y9uhIAW1fIP+OnfamOF5agmcMe3Nrqcev/lc+p69notHJV8xLZ6kMnrAzuVfv8Nlzw2P1ozpVGFI0bPQ48sQ6bTnzb8537Mu5Vr0airMB0NnWVEH30xmDPXvesZUpE1Ei1JDmJY4CFfxnRhVmq7LB5IapS2M0YlbYU1oEWt6PLNA4cUAvhEsRYY/CgKxXV/mddUxo+6nccG4PWl8251+XvH+lF/3FJPogqAskev+ZcWiGqu1cqRSF32Kr0aZr8pXL8ymr7x7HaSJqTOCJs8y8juuiLqmTVlpOwDrS4mdth1YrX2emBsQ5SfvvZ7LzL307stmMD8v0/9g/xhvbt7QQe/IJ+xdMyQ/rd6hdT5PtnJ8nHY2r31t0rnbDsudPchi47xxU/AG58xTv2OEooJOU7HhORSzqXi3oo6Txn9cOq4rG4dfm84GV3k06SGl3IkrVeeMw7Nl1pYjdrj+jPF6QkcpxsmKOPVCqMw23zB4CPN/0C6zN3WKYh1/6BSjkHyd7d26lP1IojE9hNXKWck/QdV1DJks6MAl3phSCO1qTU/PelCrHKSkzHYlQqAv4VKCM4A+3nu/D3j+OmpvvwhzgECiJYdILfUVu/vbDJ+V1aeuF1GJc+K3opZ2e4r9+x5nbIWyIWj+WYXOIStnGZbJL0HY8IC/8y4ld6wcTRWpVa+9WmktqVy5E6msoglR/yGoP8KlCGbIM4+OgaLBt8A/9NnIQ/yBxHE0YKn4up1oxLg5Z/52Io5ewM9zWxq1+8Qa/9r22xkt+EIykuyj1JYinnKsM2/zIStR0fUJ1yE1Wl3LHUqgqVK5/Ds+/fgKFRV2qSAH43+Wx/J2TINojNudeRIoHJqaNoopHSfUxt0qpkJ10SVFDs+3bkIJTVUlWrH7uV5toWy0TlymZW4hT8zuOFKXWcxFLOVYY1/zITNWy0KrX2q0k5Y6l9NNOZv96ErKvwGhEg3n7J/9hBNUtpMpoE1crBuTrKToSlxzmEZSpjaddx4PFnCBTzIZwtG1VRPE5be+6w5ZwGrM+GCcsNG1pZRyabOGDNP+HUXVSQH0HMJyZZos59VGUdCprke4S8gdB7hEHDuaCapakAk60c3Kuj3GEgPa6g6RfO3X57fIJOlTVtO2btss5kKE7stpwmDWFk1GhoZdJgzT/hxFHzv6YwdczJtPitn7MciXaMt3sfVUx/QRC/SVMxDd4J4E2agmnO86r8EXG2QQTUKweZMB4dssod3PiK2fmDYOLkfej6YGUzjvR6E9co5X+MBrfTxwlr/gknDr9BTWEaS60ym+QOj/kITE0rlALWtuLk1AkMiVJ9KCeacPD9q6wXcfojJNd5QqRxWJyMvLDaPnpWDiV2dwlBzCFBauv4+TNM77Pss86ifJd/X78SyE5qeDt9nLDmH4FKhWDGWW4i8ZiWfNAJOtuUYyoMC9rmyfnf4wQsAdyKY+gXk/GL2dehY9m1lnB84PNezdTpgPQbs3vVcNY16P3Vg5hOb6NfTMbGkQ5055cCsCzqr8y9pPSzfiGpJuaQvZ3eMEu/KBo/f0ZQG3y6Sa69u5v02BnYTr+CfQ0Jj7WvBTjJKyTuEEygDpK0agnf5CwKXeenNz8FS4e+DcBaae346CF/wZvJ6hO8FElga8W1uOvoYs/h2lqz2LH63LENftdrklDmN4Hokqt0AjdIopys9EIQqplMVyOYJnmx8A/JkvVPShO4PD9apjyYCLLz1pi1/HMhhBWf0i+mYNNIB7419SG9cFMWanMIU4WAHMyeggVHb5MrEekd/q0R7fOYaL8mQtr0WE5MViWpTDxO6DgyjOscU+HPNv+QNFwIZtLQNfSwTRJzO8wjUBwQodgqcX3TD/UCM93k60j2/O2gOfeG3KeT3lHqX1DhjLjxw8Q8E8aPURLppCA/7B+fb+KHqLPKmtWEbf4h0ZVuYCqEHV2jM0nIEoUCkMUJ9ZuUspYJKpw2eE0Uk9Snc6uBEzVo5IupGSxMXoUz0klVR8cn29koO7jOKmtWE9b8Q1LJjl+MD86IEbcWrNNGs5OAhZ8pxuYHM4ASMKHV0mhVDB0b016DVoT002QpPSakTbX0ID10o2jSYTq0mVbdrLPKmtWENf+QVKrjFxOBvZ2WAFYxkrOafBdKN1AQxyUEkHtHv4sddvraM1ZF0eGcOoLFjUrDzU6yxh2mLv0Lj+nfd58fMIuskVVQDdqvwNScM7fDup8777LuI6Wt/gh+184RQh7Y4cvUJyZOSKDUUSj9jKuto/NzgOFkIWkNGSYyJ5MFxmXl1TDd1yETdMqyxi7s8QH+kTWqcc5YDPzH9jEBveDT+vpIpo7cMNE+DRYhxA5fprEJUztHVqJh4V+rzQzG5Qk0jcxtTBvBqFYb7mxbWSKazuxil6x2lqUwMcWo9nnl6TFHuBi1VgI681SU5D6/Ym9xNHKpQyIJfyKaRESPE9ELhf8nKvYbJaLdhX/dUc7JMEaErZ3j9h+UdN1Cqa0dKBXQfjX/VeNTCWzA68sIk21rj/e8NVbIpQwxWhol5R6jauzK+2ww4TkxrY0UJtqHI4SkRNX8VwN4QghxKoAnCq9l5IQQ8wr/lkU8J8P4YxT9QZag9StvMLdjTDO1tVmZgJaWJ1C0ibTr39/6p1Yugqlm6qch6wTd3A5g/B8oL9NzTt1E87OvALdMgpEZyTkGHTrHvcmYVIT5TAMQVfhfBuDuwt93A8VmQQxTXWRCMt3kyAtw2OFNYttNTAdu7TU7Ccg0y483eNi/7+6Rg96Ydz8N2U/Q+TmpnQJaFR307utWU3tlETbNhBeVMNE+HCEkJWq0zx8KIV4HACHE60T0HsV+E4ioB8AIgPVCiC7ZTkS0AsAKAJg1a1bEoTENjaxOjF35EvA6Tf1i23UatczBCugdzsOaKKQSXGYgO55eNU6/Ojx+sf5Zh+VWFR2UH5FvB6zJKEy0jymmtZ+ifqYB8I32IaKfA2MVbR18FcDdQohWx77vCCE8dn8imi6E6CeiPwLwJIDzhBDaDhkc7cPEgmnUDwBtM29VNIo79BLQR+VEwbSEgS6s0e9+OMswmEYHObGb3HNoZdWIrYG7EOIjmpP8johOKWj9pwB4U3GM/sL/LxPRvwCYD8CgPRLDOAgjUIKUG7bt8LJzqDRqQG4OClri2ARTB6VuZWBvVzVQt8swzO2wVgFBJjCnw5u7ZiWeqDb/bgCfKvz9KQAPuncgoolENL7w9xQASwA8H/G8NU3Xrj4sWf8k5qx+GEvWP4muXX3VHlLyCVtL3zRpK5O1zBWqc8js+ZSKV7vPZIErfqDOSo7DZm5Pbrpx26asE+8GO7Zfk3smUUQV/usBnE9ELwA4v/AaRLSQiO4o7PMnAHqIaA+Ap2DZ/BtW+NuloPsGchAA+gZyuGnrPp4A/Agbq60Kv6SU12lqZ+GqzmFHo1yxGRg6qs8ezk7yzwFIZUpbL9qJVbLj+tnMTVtaFic3DS0zrGvWla4ogawyGX5N7plEEcnhK4R4G8B5ku09AD5b+PsXAM6Mcp56YtO2AyXlewEgNzyKTdsOcGkIHWFjtVURKSLvtZ+rGpC7z/HEOst5rCKTHWue7jQhTfqj0qzX9/9lqcBU2eP9auCbFkUzNYGdt0bTjN1FdlJ5WkcyZYczfCsMl4IOSdhYbaUJRbLd9Bx+E44deumMWz9vDdD7K33Wq0o4N52kF/w+jemNx+08l4mJyTnJMTUHC/8Koyr5zKWgfQgbqx3kc6b7ahOKZsoFtYnZKujqxq9xuvtzJgJ9aND6X5snocnAZWoGFv4VhktBh8DZjF1Wg0aHadmAIPuet8YShG5SmeAlmp3bg65u/Mw4WVfUtUktImdjdfe9uOx7lolHl4HL1Axc0rnCcCnogLjt2bIaNH4ECTt072s7UmXhpc5wST+7vCps0img/RK03Pg5bk+8OxapZF8bMJb45q42mspYzua1rWPXWq3WiJwnUHa4pDOTbKrZszXOUsAb5iiEv8thair0fvYVq8SCH6ZN2bMTrQgmpyO7WmWPG6wEc9xwA3emPlBmmWqyceMizoknzuvY21mIxjH87V7xg/DN3SvZGL04GSlWNNyk3Qiu58/UB9WsyBhnKeA4r+OJdQhUdmHr56yVhy4hrtplj01yEBq8BHPcsPBnkk2UiowmiU864hTYcVaWDCME7ZaSqntQ7bLHJjkIDV6COW5Y+DPJJki0jpOw5SCcxCWwo0QryQgrBHUZ0dUue+w3oXEJ5thh4c8kH5MmH25M4ur9VgZhJx4neztL6/aLUStM9NQLrLGEWZVIQzYLJRZUSW02KiEb5Fqjrqhk+OVOsLM3djjUk6lP/GzYpiURolanfPRGbymI0aHSSB3VuVXo6tP7lWzWVS41uVbT+xYUWZirOxSViRWO9mHqE7/olUpFt6xtMd83rjo5ezvlJZszWeCsa7yNVmwh2zLTP56+nPetJNrHJfg51NMYjvZhGpsovW6rRe5wPCaUuR3WJFIsD+1TuTRIO8ty3jfbvNcyEx6N36SCKxMIFv5MfRK1121QVHbwYs9gQ+IUcDJfiZ+Q9hOylYgKSuLEXIew8GfqF52jOM7oFl1k0cUbrLIJpugEXByOVhMhrRtDJaKCqh122iCw8GcakzgieWx0kUVzO6yeuKYrAJWAiyN0FQhW3E1GnPctyBg51DN2ONqHaVzi6jPrZ6aY2+HfOhHQCzi/CcYUv+JuJkK23P15ddFMTGyw8GeYqLTMUETAODRorb2a/AVcWezgNFZVNPdOsoQsN4AvOyz8GSYqp14A9NwJrQatnCAMQyRNJhgT3HH6ucOFxvGbWdg2GGzzZ5go7O204uZLQhPJiqeP08Eclx3cJPOZaQhY+DOMmyBRNdKCZMKKp3cS1VEal6OVwyiZAmz2YRgnQcsXKIWpxEQT1Y4dhx08LvMRU/Ow5s8wToKaRZRCk+LJ1o0bDqNkCkQS/kR0FRHtJ6I8ESlrSRDRRUR0gIheJKLVUc7JMGUlqFnkvDWwwiXdiGTa0SsRp8/UBFHNPs8BuALAP6p2IKI0gO8BOB9AL4BniahbCPF8xHMzTPwENYvM7bA6ZclIqh2dwygZRNT8hRC/EUIc8NltMYAXhRAvCyGGANwL4LIo52WYshHGLKKqoc92dCbBVMLm3wbAqUr1FrYxTPIIYxaphB29HA1UmIbG1+xDRD8HME3y1leFEA8anENhEJWeawWAFQAwa9Ysg0MzTBkIahYpdzmCcjVQYRoaX+EvhPhIxHP0AnCui2cA6FecazOAzYDVzCXieRmmcpTTjv7ojfHU9WEYB5Uw+zwL4FQimkNETQCuBtBdgfMyjJ5aMKXs7VQXhEuqQ5mpCaKGel5ORL0APgDgYSLaVtg+nYgeAQAhxAiALwLYBuA3ADqFEPujDZthIhJXieRyE6axCsMYECnUUwjxAIAHJNv7AXzU8foRAI9EORfDxEpcJZLLjV9jFYYJCWf4Mo1JrdS4UWn32Uljk1QtmK+YxMHCn2lMaqVVoCqM9OIN1t+1Yr5iEgcLf6YxqZUaN355B1yimQkJV/VkGpNaahWoCyMNY77a21kb182UFRb+TONSDzVugtYi4oQxpgCbfRimlglqvmIzEVOAhT/D1DJBaxHVSpQTU3bY7MMwtU4Q8xV38mIKsPBnmHrG7dw99QKr4bzT9JPEKCem7LDZh2HqFVkOwJ4fA2ddw528GNb8GaZuUTl3X3gMWPlcdcbEJAbW/BmmXmHnLqOBhT/D1Cu1UsKCqQos/BmmXsbUcZoAAAMLSURBVKmVEhZMVWDhzzD1Sph+xEzDwA5fhqln6qGEBVMWWPNnGIZpQFj4MwzDNCAs/BmGYRoQFv4MwzANCAt/hmGYBoSFP8MwTAPCwp9hGKYBISFEtccghYjeAvBqtcdRZaYAOFTtQSQEvhcWfB/G4Hth4b4P7xVCTPX7UGKFPwMQUY8QYmG1x5EE+F5Y8H0Yg++FRdj7wGYfhmGYBoSFP8MwTAPCwj/ZbK72ABIE3wsLvg9j8L2wCHUf2ObPMAzTgLDmzzAM04Cw8E84RHQVEe0nojwRNVxkAxFdREQHiOhFIlpd7fFUCyK6k4jeJKKGbr5LRDOJ6Cki+k3hd/E31R5TtSCiCUT0KyLaU7gXtwT5PAv/5PMcgCsAPF3tgVQaIkoD+B6AiwGcDuATRHR6dUdVNe4CcFG1B5EARgD8rRDiTwCcDeALDfydOAHgXCHEWQDmAbiIiM42/TAL/4QjhPiNEOJAtcdRJRYDeFEI8bIQYgjAvQAuq/KYqoIQ4mkAh6s9jmojhHhdCPHrwt/vAvgNgLbqjqo6CIujhZeZwj9jJy4LfybJtAE46Hjdiwb9oTNeiGg2gPkAflndkVQPIkoT0W4AbwJ4XAhhfC+4jWMCIKKfA5gmeeurQogHKz2eBEGSbRyexoCITgZwP4AvCyF+X+3xVAshxCiAeUTUCuABIvpTIYSRX4iFfwIQQnyk2mNIKL0AZjpezwDQX6WxMAmBiDKwBP89Qoit1R5PEhBCDBDRv8DyCxkJfzb7MEnmWQCnEtEcImoCcDWA7iqPiakiREQAfgjgN0KIf6j2eKoJEU0taPwgoiyAjwD4rennWfgnHCK6nIh6AXwAwMNEtK3aY6oUQogRAF8EsA2WY69TCLG/uqOqDkT0EwD/DuA0Iuolos9Ue0xVYgmAvwBwLhHtLvz7aLUHVSVOAfAUEe2FpSg9LoT4memHOcOXYRimAWHNn2EYpgFh4c8wDNOAsPBnGIZpQFj4MwzDNCAs/BmGYRoQFv4MwzANCAt/hmGYBoSFP8MwTAPy/wEwani1EnF5GAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(X[y == 0, 0], X[y == 0, 1])\n",
    "plt.scatter(X[y == 1, 0], X[y == 1, 1])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(375, 2)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 使用Bagging（放回取样）的方式集成学习"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "\n",
    "# 使用决策树训练500个子模型，每一个子模型中最多有100个样本, bootstrap为True表示放回取样，False为不放回取样\n",
    "bagging_clf = BaggingClassifier(DecisionTreeClassifier(), n_estimators=500, max_samples=100, bootstrap=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BaggingClassifier(base_estimator=DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
       "            max_features=None, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, presort=False, random_state=None,\n",
       "            splitter='best'),\n",
       "         bootstrap=True, bootstrap_features=False, max_features=1.0,\n",
       "         max_samples=100, n_estimators=500, n_jobs=None, oob_score=False,\n",
       "         random_state=None, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bagging_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.904"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bagging_clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "增加子模型数量，查看训练效果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.904"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bagging_clf2 = BaggingClassifier(DecisionTreeClassifier(), n_estimators=5000, max_samples=100, bootstrap=True)\n",
    "bagging_clf2.fit(X_train, y_train)\n",
    "bagging_clf2.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### OOB(Out-of-Bag)：大约有37%的样本是取不到的"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "放回取样导致一部分样本很有可能没有取到，平均大约有37%的样本没有取到，不使用测试数据集，而使用这部分没有取到的样本做测试或者验证。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BaggingClassifier(base_estimator=DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
       "            max_features=None, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, presort=False, random_state=None,\n",
       "            splitter='best'),\n",
       "         bootstrap=True, bootstrap_features=False, max_features=1.0,\n",
       "         max_samples=100, n_estimators=500, n_jobs=None, oob_score=True,\n",
       "         random_state=None, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bagging_clf3 = BaggingClassifier(DecisionTreeClassifier(), n_estimators=500, max_samples=100, bootstrap=True, \n",
    "                                 oob_score=True)\n",
    "bagging_clf3.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.92"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bagging_clf3.oob_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bagging支持并行化处理"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "不使用并行处理，耗时为："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 422 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "BaggingClassifier(base_estimator=DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
       "            max_features=None, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, presort=False, random_state=None,\n",
       "            splitter='best'),\n",
       "         bootstrap=True, bootstrap_features=False, max_features=1.0,\n",
       "         max_samples=100, n_estimators=500, n_jobs=None, oob_score=True,\n",
       "         random_state=None, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "bagging_clf4 = BaggingClassifier(DecisionTreeClassifier(), n_estimators=500, max_samples=100, bootstrap=True, \n",
    "                                 oob_score=True)\n",
    "bagging_clf4.fit(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "使用并行处理，只需要添加`n_jobs`属性为-1，就可以使用CPU的所有核心进行并行计算，耗时为："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 256 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "BaggingClassifier(base_estimator=DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
       "            max_features=None, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, presort=False, random_state=None,\n",
       "            splitter='best'),\n",
       "         bootstrap=True, bootstrap_features=False, max_features=1.0,\n",
       "         max_samples=100, n_estimators=500, n_jobs=-1, oob_score=True,\n",
       "         random_state=None, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "bagging_clf5 = BaggingClassifier(DecisionTreeClassifier(), n_estimators=500, max_samples=100, bootstrap=True, \n",
    "                                 oob_score=True, n_jobs=-1)\n",
    "bagging_clf5.fit(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "使用并行处理的速度明显要快"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 针对特征进行随机取样"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "如果只针对特征进行随机取样，只需要将`max_samples`设置为500，总共有500个样本，每次取500个，取出了全部，所以相当于没有对数据集进行随机取样，然后再对特征进行了随机取样。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.826"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random_subsplaces_clf = BaggingClassifier(DecisionTreeClassifier(), n_estimators=500, max_samples=500, \n",
    "                                          bootstrap=True,oob_score=True, n_jobs=-1, max_features=1, \n",
    "                                          bootstrap_features=True)\n",
    "random_subsplaces_clf.fit(X, y)\n",
    "random_subsplaces_clf.oob_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 对样本和特征都进行随机取样"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.812"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random_patches_clf = BaggingClassifier(DecisionTreeClassifier(), n_estimators=100, max_samples=500, \n",
    "                                          bootstrap=True,oob_score=True, n_jobs=-1, max_features=1, \n",
    "                                          bootstrap_features=True)\n",
    "random_patches_clf.fit(X, y)\n",
    "random_patches_clf.oob_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 随机森林"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = datasets.make_moons(n_samples=500, noise=0.3, random_state=666)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD8CAYAAACfF6SlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJztnXuUVNWd77+/qq6GajKh5ZHRbvARh8HxgfaAjwzMTNRRYlRsMcFX7iQzGm7GJEbNBTHXwZblJCjrqjGPSYhmTFY0oRVs8bUwUZNcTMwFbECJMhpNpLs1AbTbSBd2dde+f5w61adO7b3PPo+qOlX1+6zFAupxzq5Tp35779/j+yMhBBiGYZjGIlHtATAMwzCVh40/wzBMA8LGn2EYpgFh488wDNOAsPFnGIZpQNj4MwzDNCBs/BmGYRoQNv4MwzANCBt/hmGYBqSp2gNQMW3aNHHkkUdWexgMwzA1xbZt2/YJIaZ7vS62xv/II4/E1q1bqz0MhmGYmoKI/mDyOnb7MAzDNCBs/BmGYRoQNv4MwzANCBt/hmGYBoSNP8MwTAPCxp9hGKYBiW2qJ9M49PT2Y82m3RgYzKCtNY1lC2ejs6O92sNimLqGjT9TVXp6+3HDhheQyY4BAPoHM7hhwwsAwBMAw5QRdvswVWXNpt0Fw2+TyY5hzabdVRoRwzQGbPyZqjIwmPH1OMMw0cDGn6kqba1pX48zDBMNbPyZqrJs4WykU8mix9KpJJYtnF2lETFMY8ABX6aq2EFdzvZhmMrCxp+pOp0d7WzsGabCsNuHYRimAWHjzzAM04Cw8WcYhmlA2PgzDMM0IGz8GYZhGhA2/gzDMA0IG3+GYZgGhI0/wzBMA8JFXgwTEdyXgKkl2PgzTARwXwKm1mC3D8NEAPclYGoNXvkzjIOgrhvuS8DUGpGs/Ino+0T0JyJ6UfH8R4loiIi25/+sjOK8DBMltuumfzADgXHXTU9vv+d7uS8BU2tE5fa5F8DHPF7zf4UQJ+X/rIrovAwTGWFcN9yXgKk1InH7CCF+SURHRnEshqkWYVw33JeAqTUq6fP/CBHtADAA4H8JIXZV8NwM40lbaxr9EkPf1po2igVwXwKmlqhUts/zAI4QQpwI4BsAemQvIqKlRLSViLbu3bu3QkNjGAuV6+b0Y6YHjgUwTFypiPEXQrwrhHgv/+/HAaSIaJrkdWuFEPOEEPOmT59eiaHVJT29/Zi/+mkcteIxzF/9NBspQzo72vG1xSegvTUNAtDemsbXFp+AZ17ey2mcTN1REbcPER0K4I9CCEFEp8CadPZX4tyNBhcbhcPtuunp7Ze6ggBO42Rqm0iMPxH9GMBHAUwjoj4ANwFIAYAQ4jsAPgHg34hoFEAGwCVCCBHFuZlidBkrbPz9YU+kKjiNk6llosr2udTj+W8C+GYU52L0cLFRdMgmUhtO42RqHZZ3qDO42Cg6dBPm1xafwDsppqZh419ncLFRMWGC36oJs701zYafqXnY+NcZqoyVRjRWYeQaAJ5ImfqGhd3qEC42sggb/I6yape1/pm4wcafqVuCBr+jNtScfsvEEXb7MDVBEN99kOB3WFeRDNb6Z+IIG38m9gQ1yKcfMx3keszLZ18OQ83pt0wcYePPxJ4gBrmntx/rt/XDWUlIAC6aq4+HqAxy/2AmsFQGp98ycYR9/g2IH592HAKVQVbOsglDAHjm5XHBQNlnUyl7AsF99csWzi7y+QOcNcRUH175Nxh+XCjl8H8HIcjK2WvCUH2204+ZXpLe6SSTHUPXxlI1cl1MgtNvmTjCK/86x726HR4ZNU5/DJsqGXbXYL+/fzADAopcOF4rZ502P6D+bD/+zR5ceupMPPPyXuUOYDCTRU9vf+GzmGTzcPotEzd45V/HyFa37wxnpa+VrZTDBCrD7hqc7wcsw28Hb01Wzl4FWqrPMCYE1m/rx7KFs9Gu2Vk44w2czcPUImz86xidMJkbmQslTKAyrEFU+ezbW9N4dsUZnqtoL1eL7jPY49TtLJyTh1eQmPsqMHGE3T41iolLxTSVUOVCCROoDJveGEV6pM7VIvts7vN0drTj5kd2SXdLzslD5WIioPA4F3YxcYNX/jWIqUtFtbolAIe0pDyDj2EClWHTG8udHml/tiS5KwEsJqdTmL/6abwznPWsFZC5mNwxCoBdQUy84JV/DWIaiFWtbgWAg9kc7rj4JCP3SZCVatj0xkqkR9qfy32eVIJwYGQUgxlrxW/HGwSAJFGREXdeH+dOjLt/MXGHjb8P4pDzDpi7ROyxfbl7B8ZcjdMy2TFcs257wbcd9ecwEUXTXc8oRdX8jnN4ZLTE1WNPAPZ17B/M4Np127H1D2/jls4TiiaBnt5+XLtue8nKH+DCLiY+UFy7Kc6bN09s3bq12sMo4E7nA6yVaDXytU+6+cnCqtSJHQx1c9SKx6SGyCbM53Aa8NaWFIQAhjJZqbG+secF/Pg3ezAmBIgsY5pzDEw7jp3dwFOrgKE+YPIM4MyVwJwlvsdrgtf1ckJAyQ5q/uqnlTEAk90Ww4SBiLYJIeZ5vY5X/obEpTduT28/DoyMljyeSpDSJaJzQwDW5/hy9w4A6mCkbJUOFLtMnKtld4Dzxp4X8KPn3ig8L4TaJ14yhp3dwCNXA9n8ZxjaY/0fKEwAUe7KvK6XEwGUjFm1MxMwC/bGZYfJ1Dcc8DUkLuJcazbtRnasdF36gYlN2swWXdUqYLkzdJW+yx7cURRgXvbgDnRt3KVNJXX6xn/8mz3a89tIr+dTq8YNv002Yz0O7wC4X0VQVQDXdMwq105rPoisG0dcqqqZ+oeNvyFxEedSTTaDw1mlkXNm7ehQZaPc/MiukgknOyakrifVeN0xBxXS6znUJ39x/nHdriyIMZVlOV1+2uHKCcA9ZtnkYQeRvSaoa9Zt54IxpiKw28eQuIhzqVwSrS2pEokBWUBSFrtwIptcVFXBpuMFrCwZrwlAeT0nz7BcPbLHod+VBXXXqbKc7nvuDU+ZCdMgstOo674T3WdkmKDwyt+QuIhzqWQLhIA0pfO+596Qioyp8tv97mR07iSnYbz01JnS17SkEt7X88yVQMo1rlTaelwz5rbWdKTuuls6T8AdF59kdA90drTj2RVn4PXV5+LZFWdgUCOrYVKJ7dWAhiuJGb/wyt8HcRDnUqVAXrtuu/T1dkDS/Z5LT52J9dv6jXYyremU0sUzMZXAhKYEhjJZpFMJZEZzEMJa6Tu182/pPAEACtk+SSJceurMwuNa7KweSbZPT28/hl0B8EWJzbg+1Y22g/vxx4nT8NWRT2JjbkHRa4K664LeAzqhOa+JSLfD5BaRTFAiSfUkou8DOA/An4QQx0ueJwBfB/BxAMMAPiOEeF53zLilesYdVXqhTTqVLDH0F81txzMv7y3J4JFl9Sx7YAeyOfm9Yh9LNpnIVsZRZbPIXFiLEptxa/PdSGOk8FhGNOP67JWFCSCKFF2/n0GXKmwrl8po9zi26ntXpf0y9U+lUz3vBfBNAD9UPH8OgFn5P6cC+M/830xE2Kt/mXm2q1KdZLJjeOblvUUGQrWK/NriE7DmkycqjZQthSwrJHPvOlpbUvjoyM+xLrEObRP2YWB4Gu586BIAV/k2xjJ3yfKm7iLDDwBpGsFXmh/AIwcXFE1o81c/HWgCCrLa9ipaC1pDEpcsNKb2iMT4CyF+SURHal5yAYAfCmub8RwRtRLRYUKIN6M4P2MZl61/eFsakDQN7qqCo10bd2H7TWejs6NdWQClCubahtE+7t8ffAb/kbobLWQZ6Bm0D6vEWtz2WBM6O26WHkO1ypYZuDbaJz3GodiH11efWzheGFdJVEFk21c/MJjB5HQKE1MJDA7LC+RUePUtYBgVlQr4tgNwpmv05R+rKvUWKFMFJFUpnm4DoUwjzTcvkb3HRhVAdu86ljd1Fwy/TQuN4MqRH0nf39Pbj2UPuGoMHtiBnt5+6VgGxDTpcezMICC83LROwjlIvwIB6xrbeksmktU2Xn0LGEZFpYy/zDKULBWJaCkRbSWirXv37pW8JTrqtZjGnWXS2dFubCB0q0XbMKqOdempM6WPu3cEqpV5W2K/dCLu2rirJNaQzQl0bdwlHcuduASjyYnFB3dkBgHhXSW662R6D0XVACYuWWhM7VGpbJ8+AM5cvxkABtwvEkKsBbAWsAK+5RxQXOQaKoGpSNqyhbNxjSJryDaMumPNO2JKyePuOMGAmIYZkgngTTFVqn2vyjIazGSlY1mw8Co0JU/U6gAFcZW4NYxSCZIGwJ33kC4oHKWvPg5ZaEztUSnjvxHAF4joJ7ACvUPV9vc3WqDMxECYNi9RHUv1uNO/ftvoEqx2+PwBIINm3JotFmkzXQXLz7lEK/qmkroeHhkt6s1r444RvDOcRTKhFnwYyLt/dHEF9tUz1SYStw8R/RjArwHMJqI+IrqCiD5HRJ/Lv+RxAK8BeBXA9wBcFcV5wxAXuYZq4457nDvnsEh9yG5picfE32NF9kq8hekQIGDyTKwYubIkDx+wjOghLSnpcQuP7+wG7jge6Gq1/t7ZbTym1nTxsd8ZzkrdNrJd4pgi7RWw7iEvtw776plqE4nxF0JcKoQ4TAiREkLMEELcI4T4jhDiO/nnhRDi80KIo4UQJwghqp7AX4s/vqgD1LK4x/pt/bhobnukPmQ77pBKWBIPG3MLcNrBr+OvR+5Hz0c34RcTTpe+b3I6hZvOPw6pZPEqO5Uk3HT+ceNqn0N7AAhgaA+G138eXbfc5HltOjvaMWlC6cZXtuPwsxu07yGvnWW5fPX1lsTAlI+GrfCNullI1DK87uOdfsz0oiKqKCo5VavTx3a+iZbmaG8NXeBWkSgEIo/v6Y5StU87c+isDR8BoL82Xo3X7fO1NCdxYEQvv2BjG3BVTYSJ+ywoXO3L+IGbuURA1I1eZMeT9YQFxitAg0w8pk1LoqiIPXLFY8rnVJ+NgEJuvpue3n4sevg4JCTvzAnCh9+/z7PKVdd0Jcivwnk+3XfoVbUbFK72ZQDzCl8WdguIc3v95e4dxml7Jtty2YpcZYz6BzO4Zt32QCmrpvGNcksKJ3yKzNmGdSA3Vfr8gLAet1fwqmth2njdBLfL0B3rcB63XGnFjZbEUHECxJfiDBv/ALh95arqVvePzrS2IOyP1dRYmzR5iWpMqsAtIL9+uviLPTneNroEw6K56Llh0YzbRsczfXSNXdZs2l0S3wi6D5btjOyaC9lx/U6oJosGTmIoI5L4Eh65uqYnADb+ATCR4AXGf3R+G3Wofqy6blJuTIy1KutFRlgDIgvc6nAqgrqxP9vG3AKsyF6Jvtw05AShLzcNK7KlmUP2Nb6x5wVc69olrd/Wj2ULZxeK4rwa3shob00Hii2YTqimi4ZaTGKoGTy6ydUibPwDYPKjtX90zh+u6fFUP+LLTzvc2DiZGmtV1ov73GENSGdHO9Z84sSiVbaOZ15WV3g7P9vG3AIsGLkLH37/Pvxj9hvSlFHAMphu3SOgdPL1sxuysesDTMZr8rgb02pgrvYtIx7d5GqRhs32CYOqQCdJhJwQRUHX+auf9t2owysTyUS+Waf/7j6u12TmNDRehkSX9eTObtF9Dt2YVF3VdPLIuk5iznPJrv3px0wvSF+3tqRwMDuGTDZXeM87w1kse3BH0fud16J/MFMSS/AzofrZOdRTtW+sGtl7dJOrRdj4B0BnfNw3Z9BGHbofse6Yh7RYufFAqWQxgJJUwGvWbUeCAK+kL5O0Qb+phjoZat2qOIg8sm4Clk2+OiPTserJIuMPWD2Nb35kV+F9N/a8ULTTEAie7dOI1cCRpa3u7NZKfRhz5krLx+90/bg0o2oNNv4B8FMjoPrhAsFT/pR9fNMp9K48W/nDmdCUkBpBTbFqEV7aR371knQy1F6rYp3EhD0WncaQDQG+XVqqnsb24z29/VIXk234/aZdmvaPjtVKOSSRaG/ZQVrbYNtBWsD/BKDpJlersPH3QPWDMt1e+9klmJ5bdcyuRdaKX/XDMQlSe6HbdQQJbN7SeYJUEC5sRbHb/XLg/dGS1xGAy087PPJOY2s27VZmDQUVbrOPqxpTvRV4RZK2qgvSBjHac/SaUbUGG38NUfygglYSm5zbr2KkKelUEhOaElJFTZ2rIah7Ioyf2stQy4qtgHH3mJeIm+y6q3oaE7wL58rVO7jeVGojcXXVYZA2Stj4a4jqBxXEuHmdW3dM1Q+nJZVAJpvzzGXPZMcwMZUo+MoXJTZjeVM32mg/DtKhwE75ymnZwtklvX5TCSpLqmFPb3+JAqlqgpTteISwnrt23faiScPkO+9adJy0p7HXdQ3iYjKl3gq8TF1dWuowSBslbPw1VPMHFeTcuuwSW3/etIhpcDiLOy4+CdsfW4vl2XEJ5pbMm3q/qTuV309xggIvnSMnbkOt605mr96dk4ap3s/Fp8wsZAAlNJlETlqak7h23Xas2bQ7cn98vQWFI9HeqsMgbZRwnr+GalZM+j23u57Azi4BrCDjByY2ITtmXr/ali9c6pq0vqTtoqq4Zc2m3SXnyI6J8Xz0AOXxsgKn+557Qxu/cBpwvxIWugI79xhOP2Y6Xl99LnIaw0+w3ESpJOHAyJi2SCuMImc9FnjZFdLOrnS+mLMEOP8uYPJMIC8fjvPv8ue3rzNJByds/DVU8wfl99xOd8WixGZsbr4av5twGZ6b+CXceewrygwV+7hOUknCgfdHcdSKx5Dz4TfV7lYClsf70TmycRpwP0Vb/YMZHHh/tKQSWab3IwDc99wbyl7CNq0tVnzAPSm6i7TCthWtZoFXrGWk5ywBrn0R6Bq0/pYZfpWBr0NJByfs9tEQteyzF273xkVz2wuuBdvdIfNTA+OGd1Fic1GnrEOxF5O33YhFCXnDFLcqaGtLCu8dHC24RAZyUzEjIem7K/Gbal0PT10fKPPCr4tNJrAGFH+HwyOjyslwMJNFKkE4pCWFweGsNlVX5I+rq1fQTbrOzxZFfKkaBV41n2WkSweNOlsoZrDx90D1gyqHfr/7R7R+W39h9WbaFnB5U3eJmyZNI1je1I2NI8XG3zaUTgPpNnSytosqv6k2SPdwsMwLlfGVrcZb0yl0LSrO4JF9T0BpIZiTbE6gpbkJvSvPBmBWiRxEEG6yQ1NJF2uQtZZUUelc/5rPMtIZ+DrPFmK3TwDCbtFleOm3mLYFbJM0RweANtpf8ph7YpEZOKd4mpffVOt6UGVYeGReeOkc2ed54O/6sP2D16Hz4eMKW3fV92R/dp2+kNMYL1s4Wxm3bm1JFY7plwMOTSCd68j03urp7ceyB3YUfd5lD+woqxum5rOMdAY+4D1bK7DxD4Cp0JYfvH5EXlkogGXQ/kTT5ccRxdr3TiVKL5XSjbkFuLjle3q/aR5lkO7MldaOwYlB5oVqQrml84Tx83x8Hzp2rCzyzWY2fB5PP/BN6ff05W5Lh0en4unuuHX5aYeXTADpVBJCIHDxnDMYrotNmN5bum5p5aLmZaR1Bj7gPVsrsPEPQDlWO14/It2PybmiPXTxV0tu2IxL454AnH7M+CQRVH/IFyEyL7yyPoafWImmsYPFY8YIbmr6ITY3X43XJlyGzc1XY1FiMwCrf4C9mjYKrO/sxi2vX4rXJl6O5yZ+CRckNhcmoSFJsZcf3D19vV6nQ1Z4pns8Cmo+y0hn4KPIFoox7PMPQDlyqr2KWmTPOyn4WVeUapA8Mvlf8Mh/H1N4rQCwfls/5h0xBQA889QnNEW0RihTefzEzFvSx6fQeyB6DwAwg/ZhdepuIGvtZOzrZevsKP3kjoAgwQqgf33SfwEf7wDmnKHVDDKJA7h3GCa9f8MQdUwgkqSIqMTXghzXS7OnziQdnHAP3wBE3bPXeVwvqQKVcQDUPW9VActDWlI4mM3hrLFf5Ct492FATMNto0tKMoOi+Hzlom/l0fKMJNlrc9OwYOQuAPoewQXuOF5RJToTuPZFX/2WZbLO7msa5t469t+fwLBLbRSwKru/unhOSUW0n2OXDXe2DWCtvMOusMt13BqAe/iGRJe7XK6calu0ra01jYHBDNZs2l1yXlM/tROVy+Cd4SzOGvsFVqfuxozEPiQImJGwVsi2i8Sm3H18w3B386dK2jmq1jTttK/gAjJaTXtkfMjuBdVyylb11N0zYe6tCYqYARHhhg0vSNNOq/69lqtDVh123ooadvtIkKVVLntwB7o27sJQJltYlfuV5g1yXnfOdE9vP4ZHShUqdX5WXa66LDW0RZEaGtcMjpPOXYqVD43iGvETtNF+DIipaKGDmIL3Sl5LZLmAbk3djRePPRKAx3eo0Ifpy03FxaufLlJ5tVHttFRyzrIdn+m95XyvatI5MKIPSFf1ey1XOmWdp2lGQSQrfyL6GBHtJqJXiWiF5PnPENFeItqe/3NlFOctF7Lsl+yYwGAmG1lqp+l5nSsze3Jwr+Ba0ylcNNfyF8t2KrpMEj+poZXI4PBVLZqvzOx8+DismrQedzd/Cke/fx/Ob/pPrKF/xftCXdmbphGc/LtveJ9TEhC0m8TPffenOLnnHyBclaF+gqCyvsJ+UjudqaxBqVhmjqyStlzplHWephkFoVf+RJQE8C0AZwHoA7CFiDYKIX7reuk6IcQXwp6vEpishDLZsaLOTeU8r/24LiXTKXTm3jHoirgGxDTMkEwA7tTQSmRw+KoWdfl0WzJvoiv1XXRddlzep3s2cOuPgMzb6hMO9Xmf0xEQzA31YSA3tZA5VVT85qgM7eywnpfFb5wr9daWlNYV43VveaXoAnp5bvv5imTmqCppT7wM2HF/9OJrQUTdyhV4jilRuH1OAfCqEOI1ACCinwC4AIDb+NcMOjeJk3eGs76qL4Oe116Z6RQq3bgNiD0JHLnisaLXySp47ZWtrCdxOXHrE9lB6D89PB1IfrX4h2hSep95R3/CyTPMKlTzGR9HO7T6NzdfrRa8m7NEWhnunmhMpR+CvIYAz4pmWUV0ZLgN6cgB+ff1ypNWEDZqo+u381aYrl81OmlEYfzbATidon0ATpW87iIi+gcA/w3gWiGEJIUiHui0WtxEWcYuS+d0iqyZSgfbuI1DT29/ScbJxtwCIIuCXv+AmOrI9hGBW00GQadPVPJDNPHpqvTcgcIqcOB+85oN5+SscpfpfMomK3XnuUxe4ye2UDHZB5khVTHU5z+d0tTY+jmu12JCdc4oW0VWmCh8/rLKd7eFegTAkUKIOQB+BuAH0gMRLSWirUS0de/evREMLRidHe3GPtQog2WdHe24aG47kmRd0gQVxxpkhj+dSuKQllTJ40CpAVG1F9yYW4AFI3fhw+/fhwUjdxWleZYrvuHE9rnbY5MFoUsyNUx8urICHgBITymk/PmpUHX68gfENO/zu/BzrziL8FT4iS0ElUcOpNgpM6Qq/Prgy6W0qVtM6M5Zw1lFURj/PgAzHf+fAWDA+QIhxH4hxPv5/34PwFzZgYQQa4UQ84QQ86ZP9775y4lO98VJlMGynt5+rN/WXzDyqsbqlP9jpwHedP5xRkYg6EQlTQcMo3PueO/wrcdg80PfLlrBGq2qTUrvZRWai78HXP96YVXm14DaaZhrRpcggwn687vwc6+s39bvaWjLLeMcWMPKNKNGdb1091a5jK1uMVGn4m9RuH22AJhFREcB6AdwCYDLnC8gosOEEG/m/7sIwEsRnLeseFXUAtEHy0zdAgLA7yXFSV7bepWbIGngTiqaOML6R12B2lW0FiOJXGHHoQpCA8IyBs5tvkkFp4cWEWDuEhn35Z8B7Ozw5etVufVGx0o7rDnjDrriP/f47Uk6igkgsGKnyt2WngI0T9JfL697K6yxVblvdAHiDUvV56zhVpGRVPgS0ccB3AkgCeD7Qoj/IKJVALYKITYS0ddgGf1RAG8D+DchxMu6Y8ahwlfWPtCprx+FjLPz+CZBZhuZ8Tc5n6x69KK57cq2iDZFfmSPqlctivc6K2/dPv8SarhSU2bIVfElAnDHxSdpK37LVW0OoCQ5wIn2/gtTXau6twoThyJ+YHLveY1LNTHo7nfVpFHF+9O0wjeSIi8hxOMAHnc9ttLx7xsA3BDFuSpJOZtjyFIMTfVgVD5+L3Sr3HlHTFH2/y3Z4YRZfSle46wr2JhbgCmpZnRNWi//0UXZUKPCmRqylboq5bOtNe25+i6nnr5qR2jHpJT4zbRxorqHMm+r03aTzWapoV5BXdUuUbcrCPNZqwxX+FaJIO0JActNcNP5xwU+r2pCcz7uKf4VZqureK+zriCdSuKkc5cCHTdbfl/ZlYnCp1qFTA3ZpJ9KEFJJKmr1aE+4167bLj2Ol9R3FIkIKlegUcZZUEE0XYaWClPvhXLR4nG+OhV/Y+NfJXQ/TmcLwahdTSaoctTtCeHTH7gINya/UyyjbFqYI1lFjSYn4u6mT4FGUPoZo/Spmuael7FNn7R6PCfQmk5h0oSmku/ZS+WzHAqzNu2aVNKyIVtle5HLmn1nyomFrHtD9/4aNfA62PhXCZ2P39lCMA64V6v3vncK3msexapJ69GSecvfVleyimo6cyW65ixBl+z1QSo1ZfjNPS8Tqkl/KJPF9ptKv3NZkNjZj8FLClyH1w4vzLEDI1tljxzQV2oDZt9ZIXjr3imIuunL6wc2/lVi2cLZuMZjS++k0r1ZnchWqw+O/B1+3XImnu0KIG7nZxUVlU+1nLnnPpicTkkrsu2Vuux7vmhuO+577o2CyXL2Ywiqp28ipeGWBUkSFaX9+r7/ghZnyQK1bky+szlLgA2flT9XA6mZUcPGv0p0drSja+MurSGwubHnhaIfv1bzpgxUvU9rFFvusLnnAZBlix2QKLKmEoRlC2crDfKEpoQ0FfSadduxZtPuQAqzpoHizo52tO95FG3bbsNhyPd7eHcJbtgwUnhe9XlVTXEA+IuvFC0A9qCkM4Kf72zyzJpNzYwa1vOvIl2LvIuzenr7iwy/TSV12GuqT6uqQEj1405PKUubPlmB1H3PvVEU1LX5wMQmbeaOrg1j0Aps4wl9ZzeOf/7f0U7F/R7OGvtF0f3nWRAWtjhrzhIrlbNrCFi8Nvj+6bt2AAAgAElEQVR3FrYvb5jixphR9yv/crpLwh7bZMuukmQAKrfyrorvNwi61aUqdnDOrWXx9frJ5hrMp3mGrcCW3Xuqe9Q4UPzUKqTxftFDdr+Hvx8clwHx3ElEWQkbZicYxo1Ywzo+Mura+PuSCK7Ssb1qCXQGYXI6WL4/4G/iiqRPayXQrS7tAqAK5WP7MeRemTt2u01dEZ4qTqS6R71EBAvfsaYuwzlReO4k4lQJ62fycMYpKAEI13dQ5uywclLXbh+v5ihxPbYTnWvlwMhoIMG1IJotnR3tePbj+/D6X16PZw8uRufPF8Zvy+u1uiy4Dgatv8v4g1V9b+7yKOcOatnC2Ugli19h13XYGj5+zue1GnfqAh3SkgIEShoWDacPlZ7vTUwt2vl5ugZnKbLXVI8HJUq3jFvQzW34bWo0WFzXxr+cgcpKBUF16o7ZMRFoslEZhZsf2YWTbn4SR654DEeueAwdq54cnxDKpaboIpCKpP2DVzlWfKwuA51fgkow7vLTDtcLsUmyEIFxVc47Lz7JWIjO6x51Kn22NDch61ISzGTHcFv24hIfeQYTMDB3eUlaqHtcn2j+FX5KV1mGeNu90rHglSfljwch6nvUNEOsRoPFde32KWcBTDmP7eSZl/XS1kEmG11Dd/f/lz24AwDQ+XOD5ikhCeRK80oD1ATzenr7cfMjuwqfuyWVQHZMFIxgGDdhEFfZmk27SwxwNidKmvKYHld1jyaIil07He3Ke+IH751idUfLuz6G04fituzF+MGvZqDtt6U9jMcLAf8fbhR3oymTLwSsxKrZpMGPE6/UU5OxRZgdVmnq2viXM1BZqSCol3EPMtn4EZGzdxedB8svXeurk5eNbnVmC29J3tfT249lD+4oyr4ZzuZKXhdGJ8evNpTpbtL0uCplWluewTm5aRczc84F5izxnJyLxnXH9cDQwZLjlZA+RP+8H+0lnXxDV6t3E5YNS4E3ngPOu916TBWnoCQgcjWl4yOjrt0+5dQ7L7eWuo3OuNv54X7RNXSXMTCYqUhDbHcnrxkJK72w0MlLtn1XTj6k9euv2bRbmnapG1cReVeT6GrFW11/hS995YZQbiIg+pRa+x5t1SQG2JObfU8sSmzG5uar8dqEy/DshKtx57GvFF7rK85luih4/89qt4xfN472XjRowgIBbP3++PFVaaEXfqcicaNyU9crf8D/6svOgrErGseEupWhybGDpIO6m3ynElTiDgAg76FmgMx1cOD9UWU+eVtrOjqZBRU7u/HriV/Bh8Re5JBAE7lW4bLt+85ueQYGAKn2v4Mg2ThF581fC4I1OX0tdTdWvAtcuy6DrX94G7d0nmB8fJty7Cbt+gFdrcDAYKZQzHX88/cUUjvbsQ/tL9wEHHkIMGeJvziXqUCbTpdH5cbZ8FnrOfd3a6IL5NWExSn1UMOKnSbU9crfL84sGKB0e+x3VRckq8b9nneGswABMhXdoAFfoLSt33knHiZ9XdLeXci6YkWlWZ43podiLxKEUsNv4/zB2gZY5UsGtCtF09W01PhKjJKd+y4A3PfcG4F2AOXaTZq6Dk/+3TdKcvqdhVjua2bvEn438fKSzJotR38RGdFc9Hql+GaQGgDZd+u+R5Xv7dPvEpznrWCGWKWp+5W/H3SdtIL4foNorUtVHzXuiaiyi5pefBCbm3+ENsqX8OebuP/FhKbxsZZL2TBIVoXpexQBv2ULZ5f4/AGrb/IHJ6YwlMmqd2oePQkEUJY4QdCiQl2Mp2hy8zDCzp1JScMdV8HTNb+dhbnZK/Nxm/0YEFORxkFMpfdKj+/8Xh+9zsoM0k3qNrLv1nmPKpuwzNCIvEE9MRTiD3vyfv8xbVwp7rDxd+BlSP0a2iDpoH7PEUl20c5uLM9+Gy0J64c8g6wSfmSBRzILPN4cAUGyKvwEmiWvtY2mne2zKLEZK5q7cRj2gz44A7hwpRXolGHQkyDqlN8wRYWqwG9rOoWuRccZy2c73YXLh7tLO605jPHAYAb9WICNI+P3j7RDm/N7ffQ6YOs92s9Sgu4+8GrC8sZzlo/fRCfIHSC2J6carvJlt48DL0Pq19CqXt/aklLmkivfk04Z53f75qlVJT9k241REf0e1UqLklC6mPwEmhWv7exoR+/Ks/H7yw7grkn/hTbsA5kEFiWBwGHRjNtGx8cX9XULU1QocyfdefFJ2H7T2cUTh4Huje0unJHYDyl5Yyz7/BtzC3Bb6iq161BVC6BDdx94uSrPu91cJ0i304yigXwV4JW/A13T9iCGVlVC/97B0UJuuXsFpwr6dS2yuneVRWJB48Yoq36PcxstU2rUxRRkq7pks+VYzjmCmyZBab/54Y5AoBjqQ39uasFNBijulZDtIsMWFRolPvgJcHrsElT3caFDGzB+TTYstd6nc/Us/l6whAMvV6XX80X3qIahPdoEgzjCxt+BTL9cl+3j53i6rBpnHMCriCfKhvGF4yp+yG/RVMlRIqKkOEugMAGY+FFVhgoAnrh+vPlHk8EKPEjAMW80aGc3pjyxEndm/hPLc924u/lTOOncpcXfUwSCYJUqKjSO63hkf3kWo/lprkPJ6mTemPQRcFJjLiA2/i6ibtruPt5RKx6Tvs65gitH43itz1jyQx4WzVidXYKflqtvgCrPevLMcRE2L2SGamc3MOo4buZt7x9kUNGxvHFoyX+OGYl96KLvAsnjADjO5XdnISF2yqphjbGf5jpzPzN+zkoaVT9jtKkhoTc2/hUm6AourHy0NvNohXWjvrXhK/iQ2IcB4XBj5IJXuGqJUuLXSRBDG7SGwfRcis+UG9qDgZVHoy2xHwfTh6LlnFWFSlrVdx0rZVWNMfYKUIuhPnUypp1JQ0nL8NsVt5VGdy/aY/T7vhjBxr9COIvHXN5tzxWcs5PXosRmLB/uRlvPfgw/OW4wvPD0Gc9Zgo/cP0kqjVaWvgGmq22/vvKgLhzA/yrW9Fyqzyqs3QIAtGTexOjDX0Tv79/BDVuO0Eoo1AJeAeqTxVS0077SN/rZ+ZUb5T2aH6MqlZQSpXISMYSzfSqAu3jM9m4D3oU8zk5exbIHAi2ZN41VC02kAyrascuko1IQlcagMhRBinmU3cEOKZYVnnV2yWfNCaumwEnT2EHMfH5NRaTCy41usbFm027cml2CYVcRWAYT/FeMl7Ozltc9KnseyO8Iyqd8GxVs/CuAqqtTe2saz644w1Pp0V6NL2/S5FZ7oJIYdu44TF4TGSYVw0Fa/4Vt0+cH2bkSKWDkveIJa8f9wImXFT5rX26a0uXxISFZDaOC/ZKjYGc3fj3xS3htwmXY3Hw1FiU2F55qa01jYDCDjbkFWJG9En25acgJ65qsGLlCPenKjLzJ4iDM5OB1j7qfJ4lelmkaaBXaQ0bi9iGijwH4OoAkgLuFEKtdz08A8EMAcwHsB3CxEOL3UZy7FgiTpud8TZtsmwwY+RhNfMYV9yt7BfDK7cIJmX4pPdfIgfFMI5tsxtKtz7szLl79NNYNfxYzJN/nn2ia9FSx7JcsoyDVkQGouGDwp8l/xLKFswvuz4254iIwZbOaR68rLsayjXxTWh9ziaLtop9U0a5W+Wu8fp9Vag8Z2vgTURLAtwCcBaAPwBYi2iiE+K3jZVcAeEcI8VdEdAmAWwFcHPbctUKYND3newfENKnBkLkfVEFDL0MeK79y0Cwck6yQqH5w7nMZGIBlC2fjzocuwSqxtmgnNyya8XW6FKkkFclOxLJfsgqF7tFXmh/AGRd8oXBvGWcu7ewurcIFrHOoMnHsax1BlpUvgt6vlR5nnijcPqcAeFUI8ZoQYgTATwBc4HrNBQB+kP/3gwDOJJJJlYUnTCcm2Xuj6OwUxp3ifO9to6V+UplLI4igXCw5c6VVuOVm5ED4bXEQl5IJBjGHzo52LLjwKtyWuqrY7ZG9Ej8+eBogrLaK5ZQKLxuKVe6h2FdUq+IpYGe7QTZ8FsoObSrsa12ujDIVQV2OlR5nnijcPu0AnNNdH4BTVa8RQowS0RCAqQCKlrFEtBTAUgA4/PDDfQ8kjP6J7L3LHtgB0LiwWpgm7UAwd4rzvY8MLsCUVDOWp9ahJfOW0lURRFAutsikIE1y970o1w/OMG3U2mHdjPmr/7FkV5jNCbQ0N6F3ZcT9bXWEdYHZx1BJbLsyYDo7lhQXfD11NfBw/tyzzrbiJF459ukpVk2H6lpXuml80KyxKjW3j8L4y1bw7l+syWsghFgLYC0AzJs3z+d0rzZ6X+7OtyL0CKyWqGlKNPSDGtEw7pTi954L4Gbt6yvVX7jsPLWqWKrBicm2WGfQyvWD82kA5r37U6xr7i5RU63odxWFC8xLYtsphNZz1fixZeeWuXlKIOCcW61/qq51uXtQyAhSiFaNcSIa498HYKbj/zMADChe00dETQAmA3BFxcKj+sGMCeG5YvfzYwvzwwxbrGVCxaQAyo3XKlwnCeBl0Mr5g7MNgFO7xm4+Aowbq/QhWNP8LpoxCqA4OLrtg2eFH4cpUfic/VTD5rKWBMecJepKby0EzPvX4qwbGZWShChH4kAF6gOiMP5bAMwioqMA9AO4BMBlrtdsBPBpAL8G8AkATwuhbO0QGJ1uudeK3U9f26BGNIxbyg+xkwIIilc3KFlqnY2JQXNmi6SnWCtJXaqhnx+nbPJ5+PPA2CiAfLOazNtwRzRaaATXp7qxZeEX1MeOmihcYH7dZXZGlN/3+dXPL7ckhKoX8IbP+htrpaUrEEHAVwgxCuALADYBeAlAtxBiFxGtIqJF+ZfdA2AqEb0K4DoAK8KeV4ZXb1rdin3ZwtlIJb1j0GGMaBhZXj9Uqr9wIPzkM6uKaGzEmPpYqkljaM/4D9aZkjmqmfiDFJvJJp+xERQMv4Y22l/Z7yqK/sxB3GV3HO/dwN0mlbaUPePWTUu3c4l5kVckef5CiMcBPO56bKXj3wcBfDKKc+mwfzBf7t5RaMHoRLdi7+xoR9fGXdJep0ki5IQI7aappC8+VimbNn59y/ZjD31O4UumcSPvPpZKe4WS/t0cQdwiIQLHVOZAn5stR38Rxz//78UtHP26wEz657oZ2iPP5rKhJCByxTst2Q4MqF6fXa/vOcZCb3Wn7eM7j9jBkKLJdU4IvL5a0dXJByrXkt3cJTaCXeUiiBG1Hy8xLG6FpPyxbF+yLvDo182hfDy/i/Cjd+9FKo0tR38R1zjuhzuPfQUn/+4bZTFuPb39uGHLEThr7Ip8y8V9yCGBZDYDstNeTd0WgKM5uqFXd2xE/ZzIWXIbNrLFQ89VVoPrMXk7ybJj8j3HVOitLuUdgro9yq1to3JLvTOcrf2cfBO8jKvKJSQrs1cZl8zb1vsmz5Q/P3mmfzeHbiWu2tZ7uaycUAL259pyws345y1HFO6Hue/+FMdvu9Gfy8kHtityY24BbhtdgoNoRhPlrPQ8v+dy6iOlp4QfnPu6yxYPuWzpBFLJzlom33OFd3Km1KXxB8bbzb2++lxP/Rybcmvb2JPSIS0p7etqUcjLCJ1x9fKru4XXVMYdGM+sURXc+C3G0f3AVYZGNmHNu6LUzZFsBi78buFzXfPbWUU71uVN3UgH1HMywelyDKMdFYrUpNLrm2y2ivmcC4EwAehyaecUfc9ASVZ7BVI2g1K3xj8IlQiUdna0o6XZ29tWczn5JuiMrt+KW90PaqhPL8plIirnxH697nyK9/V8dBPmT9yAo/54K+a/eB62nHhL8Xkv+FbRed3fu1bPKQKDVqTqaqod9eh1wM1TgK7J1t+PXlf6nsw75oNomlD8faSnWMV9mbdRtBAwDQ4DxQuNIAF753u9rnFhYTJk3hM4BlAZMi4jYd68eWLr1q3VHkZZOGrFY54eUVvxs+5QpUx2tULuyqFiv6+TW48qFVEDotWEd45XVb2qOJ87tRewdpK6BcX81U8XxYU2N19d0PwvQlXd6tPY9PT249p12yF053J+vkevA7beU/qaeVdYTVdMe94W4fqOVTr5ss+cSBX7/IHS66A6ntd9ImvjGOAaVxoi2iaEmOf1Ol75VwGvGEJN5uSbotLND+JvP+fW8so3P3qdlbNtrxhlhl9zviCpvU7X46LEZqRxUKJwQcDo+5FoE3V2tOPy0w4HwVA7atu98gNt+y9rMt7wWY3hV6RSu79j1U4q87b1Ge36jskzgc5vW7sn3Wo7aB1DufSfYkLdZfvUArIiLDt3JUlUZCDqMutHRpCK26IMkz2WUXD+OMOszlRqkk48injcLpxFic1WRk1mH3DH+HvdVd8XzW1H04sPYnn27lIfPGCNKXtAPqYAmSW3dJ6ACxK/wsznH0BajGAUCSSRA8k+nzKLKiffhdlMninX7JF9x14ZNGJs/H1eVb6643kFYqskuFYpeOVfBWSxhctPOxzpVLJQn1DXWT8y/Prhne+zYwlO/RinTzeIb/ypVdAafrfxkeDc4RV3YRsf45aN3y1RYF2/rd8S75Mafg+CZJbs7MbJL9yEQ7EXREATchhLTkTXgYtw1P2TitVsdVXVSsja5Z13u9l3bJJB47UCd37nIwdKA+0mO8Qoit9iDPv8Y4Lb12tTFt9/FAqOcULn01XtKLwmFmUMwgElgQu/o21ivvmhb+Ma/ATttA8yEfO3MB2nHfx6yeOvTbwcCb9SxkH90Yrr15ebhgUjVqC7EKvo/z9yn7+O9BTg+tf9vcd5jyqvgyIeJPPVJ1LAhL+wAtGm93yd+/zZ7RMTKlb9W6WuQWVFtz0PKlpmUrwjxiwf9xPXS3WBOpPP4rzU3WgaO6g8hN22seASyqt7vpObhKmJ9/Tnp4TlbgG8tYl0KK5fG+0v/LugjbXiduuBbfdan985hihxat0oJ3fFClxVD9A8yd8kVCXBtUrBbp+YULHm6fUYxNJtz4P6bVWNZGTY/QXc7qSnVmkNP2C1bXS7hGYk9uEvEget1aoOp9HVaRN5obh+A2Jq8f/thch5twM3vW1p7TRN8D6+n7RPGX7rMqL01asSFOoANv4xoWLN0+sxiKUzDmH8tmOj5mOQTaBe1zSVxp6/XYbrU6XFVc0YtdwUztz39BSEbhQuQ3L9hkUzbhstNnQlCxFTGecoeiT4iQfVua8+KtjtExMq1jy9Sl2Dyopsez7rbEe+uUsHyCTY99QqmChwFuE29jrXUT4ecTIA8byiuCrzjtxNEbRRuArX9RtOH4qVBy7CxtzfFV4iXYiYnC/qHgkm6DLH6i3eFQI2/jGiIkqcVeoaVFZ2dlt+dzvVMPM28PwPHV3ABAoTgKnGehBDastUFFJPJRtrZ8AwH39RColr3VkGE7gfQ+eYAFqG+rBq0np8oLkJP3jvFPVCRDUOmRpnWAJ+lhL1zyi09+sEzvZpROpp9bOz21J2VLV7dOKn8lcVZAQgVRRNpYETL9P3nnUHZXXn0GWVmGShPHpdaZ1C2GNG8R7ZMbzuxaiybrTfacBjxhCu8GXU1FMQS9fn142f1fyZK+UB12Szpd+y+HulPuhXntT7wJsnmVWeAlaXMRVePnBVgZouLhAkESBobYaNqebOE9dHk6Rgqr3fILDbp1Gpl9W/H4Pu5RaZdbZlwO3//+0/A7seGncnuVfu7uu1Yan3WE20goDxDCLZebzQFahF1cvAJkz7QZM03J3d6sph3y0ga1d7vxyw8W9E4pbrH2YiMm2a4o5ryK6Bs3hpaI/lwtG5Sdxj9hpL+pDic6oMv42qHsHr+9MZsLBxBFNMvlOTCUe3Evc7tjNXWv59HbWc+OATdvs0InHK9Q8jtwuo3TOJ5HhqpMwdYZKmqLomqjHPOlsjS0DjwmQlT2kkE2QG0uv7Uxow8tezIGgigOl3apKSqZvI/I5tzhKPJjOa61OHsPFvROKU6x92IpqzxFJ2dP6oKQHkxiwf++K18riG6Wf1Y3xfebK4sUdRto8msULkNJ3HJAbS6/uTauMQMO9fvXsWRKFFr7o+T1xfrLEkmyzdE45Kw9/+vv1qNp1zq+ZJUZuuz4Cw8W9E4lQEE8VENGeJlQ+/+Ht5gbd8fr5uF2H6Wf0aXzuYblr9ap/Dz8rb6/ubs8TKPLJ3FJS0DP95t5uNxwS3WN6j143/X+X6yrxdvBvYcb81Tl3g+v0/lx4n2Qwcd2GwHaNu9a/rDleHsPFvRKLc4oclyonIzy7izJVQ6svbBDW+qrHozuFn5e31/e3stgyrHVMQY9b/dYbRj/tN9tqt94z/35RsxnrfuwPW5OTeoakyuZo/IM+sMt0xlrsPRI3Axr8RiXKLHxaZzz6RCvZDVK7I95S6BeYsgadWf1DjqxuL7hymKbjuvrHOPgZ2oNWvYfTzHtOJzRQxZk0C7naQyqYu7+i/ay/idP9XEc72aVTCpOhFjVvrWKZ9bIIu20aW0TR5ZrD2fnOWAG8851C2TFruC+f11I0limIi+72yrB+VYdZNSH7cb+WKDW27t9g15ZWBJL2+ZE2AJn0g4nL/V4lQK38imkJEPyWiV/J/S6MzRDRGRNvzfzaGOSdTZzy1qrj/KmD932v7Lmsi7tUExL2SDer+MnGr6MbiXKWHQbVaV2UP2fITsiCpH/dbEJccJbwbtLhTX3Xfj9JtJxqqUCsMYd0+KwA8JYSYBeCp/P9lZIQQJ+X/LAp5Tibu+OmcFSTgazcRdxrfrfdYq3GnO8TkuM5K2vQUsxW5iYvEdi2ogot+U1qlx1BcI7vNoZNU2squUfn1/UyEJp223Mz9F+/vxj1p6dwzOrddAxVqhSGs8b8AwA/y//4BgM6Qx2NqHb95+8pUPsXjgKaJ+L3jfnOv1El7nM7qUVNNfNMJa84SK91URdjaCuVqfabcaOqCpG5Dm55iTYwblsrjJe7jz7ti/P+pSeNprpS0njvv9vHvZt4V8nHP/UzpY7o4iJ/0WKaEsD7/vxRCvAkAQog3iehDitdNJKKtAEYBrBZC9MheRERLASwFgMMPPzzk0JiqELRzlh+UTcQdj3upl4YZp59qWK9VqOkqVVYxq/uMMp+2Sn7CHoP9HpMK8DA+c9uv74yZzP2M/1TUelSorSCexp+IfgbgUMlT/9vHeQ4XQgwQ0YcBPE1ELwghfud+kRBiLYC1gKXq6eP4TFzwm4Wh6vKk6/5ESfkE4HQbeLXgC1Nf4MfoeEk+OCcMpdaQqyeBbYzPv8v6YyqNYTppebm1TM9XJG+d/85s6eTzbg9fd1DnbRbLjafxF0L8k+o5IvojER2WX/UfBuBPimMM5P9+jYh+DqADQInxZ+oApbFTZGGkD5ELd+ncPnM/I28i7nYb6FanOkPopUvjx+jIJgobd26+TmtIpdDpR5XVdNLSTeCmmlDuz2NP1lHrSHHWTmDC+vw3Avh0/t+fBvCw+wVEdAgRTcj/exqA+QB+G/K8TFypRBbGebdbfuOiCtYrvFeSO7uBW4+yMoRkht8rKOokTE4+UJpbHiR33m9g0zS/XeUzt+sJnASpBWgw6eS4EtbnvxpANxFdAeANAJ8EACKaB+BzQogrAfwNgO8SUQ7WZLNaCMHGv16Zs0StnCgzVkHcPoB/t4FX0xfbHRFVzMKvUmnQzmF+MVkpq3YIfuoHoop1MGUj1MpfCLFfCHGmEGJW/u+3849vzRt+CCF+JYQ4QQhxYv5vyX6dqSv8ZGFUSmdI1/QlPWV89R6F1lAQpVK/n7ecgU3VDiGK79X0eabssLwDEz1hc8bLYdh0xtsZcwg7Ge3sBh76nHz38NDn1LUPRrnzeXfa5JlWRfFTq/wpWvpB5tY6c6UlquYk2ey/FoAzcmIBG38mevxop1RKZ8XUeM86W/74lA97F67ZK35tKqpiJ+CVOz95piVP3TVkGc4d9wfvgRAGd89vVQ9w01gHUzW4gTvTGOzsVsci0lMsSWhA0+Tb1bRdps/j1SDcjZ+G8k5U5/F7PL9xiajOy5QV0wbuLOzGNAa2GJs7RTTZbEn8OnPSpUhSLZ+4vth4+jH8QPCgp2lcQmXcd3ZbY3e6u0xSMMvVBKhe+knXGOz2YRqH8263mqw4XSkXfMt6rhCg9YG7OYlXfwA3QYOeJnEJVdD50etKZS1svFIwTc/rp7tW2DaeTGDY+DO1hV/j4kYWyPTMsTc16qL0tam05buPMqhtEiRXpaxuu1f/Wb165no1kfFryOPUT7rBYOPP1A7lWiXqDN7kmVaXKWMVS1G8szjxsnFBtYBBz57efsxf/TSOWvEY5q9+Gj1j872D5DrFTx263YhXcD6IIQ/iSgq7AGAAsM+fqSXKJRqnlHpwBDIPP63YLz1yQO46cb5HJnHgFF5T4fCBv5/6IP5hJIdF+DMGmqfhtneX4IYNI8Di+ejUBVlVn0mli2Q9Od71TDVGXZFYEEPuRyQPMBOdY4zglT9TO5Qr4GjiRnG7i0z6wAZZCbt2NxOyQ5hCf0aCgBmJfViduhtnjf0CazbtDvaZ5n5Gs4txCcf5XVEHqZHwW+fBbqLIYOPP1A7lqgYOUmtg8p4gk5VH/KGFRrC8qRsDg/nXqFwgqvGdd7vlivKKYwQxqEEK9vxe+3ItABoQdvswtYNKc2bW2fkc9BCpgkHUIb3e49elARgZsTbaj7bWtGXoH/78eBvMoT3W/51jk43vlSehbV7vYyxFBJVYNr32O7utJjEytxXLRfiGjT9TO8iMy6yzrWrXOPqA/ej+235+A6P8JqZi2cLZwBNXyPsfP3F9NCJy5RKOC4KueprlIgLBxp+pLdzG5Y7jy985zMZvMZLpStgdxNSQwQQMzF2Ozo524GFJwBmQB6KdmBSkxc2gqtxhlGS5iICwz5+pbSrlAw6aZmoHihevtf4v64mr8/Onp+SbwFv+8PTib+LkRf8z3GeR+eYTqaLzxM6gKlNXc/EaZw3BK/8Q9PT2Y82m3RgYzKCtNY1lC2dbKzKmcgTxqwchTJqpV3qicqKicc0hGQjoIQ8AAAW6SURBVOkpii5oU/TjqcX2h5X6nhsIXvkHpKe3HzdseAH9gxkIAP2DGdyw4QX09PZXe2iNRbUloU12GF7piUGzmM651VqxO0mkrMe9MO1EFhdmnQ1p9XScXFM1Bhv/gKzZtBuZbHHwKZMd886/ZqKl2pLQJitPr4kj6AQ2ZwnQ+e3iz9757fgbcr/s7LaC+kXBcLJSVuvts1YQdvsEpJBnbfg4U0Yq0cTbT+aOGy+XRRg3TCM0MJfGREQ+ZZUJChv/gLS1ptEvMfRtraYaMExNEcZAm0wc1TbicZZV5sKussDGPyDLFs7GDRteKHL9pFNJK/+aqU+CGui4B1jjrpfDwd6ywMY/IHZWD2f7MEZUe2Wvo1yCeVERxuXGKGHjH4LOjnY29kztE3e3Stx3TjUKG3+GCUOcfeUm1IpeTpx3TjUKG3+GCUrcfeVeBNHLqfXJjikQKs+fiD5JRLuIKEdEym7xRPQxItpNRK8S0Yow52SY2FDr2vJ+9XK4325dEbbI60UAiwH8UvUCIkoC+BaAcwAcC+BSIjo25HkZpvrE3VfuhV+9nFqf7JgiQhl/IcRLQgivktZTALwqhHhNCDEC4CcALghzXoaJBeVqLlMp/I6/1ic7pohKyDu0A3Am6fblHyuBiJYS0VYi2rp3794KDI1hQlBOXaFKNCn3O/5an+yYIjyNPxH9jIhelPwxXb3L+sVJO1YIIdYKIeYJIeZNnz7d8PAMUyV0ukJhjHelfOt+dZEqJaLHVATPbB8hxD+FPEcfgJmO/88AMBDymAwTD2QpiGGzgCpZdOUnhZLz7euKSqR6bgEwi4iOAtAP4BIAl1XgvAxTHcIa7zj71jnfvm4Im+p5IRH1AfgIgMeIaFP+8TYiehwAhBCjAL4AYBOAlwB0CyF2hRs2w8SYsMabfetMBQib7fOQEGKGEGKCEOIvhRAL848PCCE+7njd40KIvxZCHC2E+I+wg2aYWBPWePvxrVciMMzUJdzMhWGiJmxg1DQQy0VXTAhY3oFhoiaKwKiJbz3uapxMrGHjzzDloBKB0TgHhpnYw24fhqlVODDMhICNP8PUKlx0xYSAjT/D1Cp+K3QZxgH7/BmmluGiKyYgvPJnGIZpQNj4MwzDNCBs/BmGYRoQNv4MwzANCBt/hmGYBoSNP8MwTAPCxp9hGKYBYePPMAzTgJAQ0na6VYeI9gL4Q7XHUSGmAdhX7UHEHL5Gevj6eNMo1+gIIYRnE/TYGv9Ggoi2CiHmVXsccYavkR6+Pt7wNSqG3T4MwzANCBt/hmGYBoSNfzxYW+0B1AB8jfTw9fGGr5ED9vkzDMM0ILzyZxiGaUDY+McEIvokEe0iohwRcUZCHiL6GBHtJqJXiWhFtccTN4jo+0T0JyJ6sdpjiStENJOIniGil/K/sS9Ve0xxgI1/fHgRwGIAv6z2QOICESUBfAvAOQCOBXApER1b3VHFjnsBfKzag4g5owC+LIT4GwCnAfg830ds/GODEOIlIcTuao8jZpwC4FUhxGtCiBEAPwFwQZXHFCuEEL8E8Ha1xxFnhBBvCiGez//7zwBeAtBe3VFVHzb+TJxpB7DH8f8+8I+WCQERHQmgA8BvqjuS6sM9fCsIEf0MwKGSp/63EOLhSo+nBiDJY5yexgSCiD4AYD2Aa4QQ71Z7PNWGjX8FEUL8U7XHUGP0AZjp+P8MAANVGgtTwxBRCpbhv08IsaHa44kD7PZh4swWALOI6CgiagZwCYCNVR4TU2MQEQG4B8BLQojbqz2euMDGPyYQ0YVE1AfgIwAeI6JN1R5TtRFCjAL4AoBNsIJ03UKIXdUdVbwgoh8D+DWA2UTUR0RXVHtMMWQ+gP8B4Awi2p7/8/FqD6racIUvwzBMA8Irf4ZhmAaEjT/DMEwDwsafYRimAWHjzzAM04Cw8WcYhmlA2PgzDMM0IGz8GYZhGhA2/gzDMA3I/wd2/CkjS78k6wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(X[y == 0, 0], X[y == 0, 1])\n",
    "plt.scatter(X[y == 1, 0], X[y == 1, 1])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 使用scikit中的随机森林（RandomForest）"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "当BaggingClassifier的Base Estimator是DecisionTree的时候，这个算法就可以称为随机森林了。另外scikit-learn中已经为我们单独封装了一个随机森林类。\n",
    "\n",
    "不过scikitlearn中的随机森林不仅用决策树在节点划分上，还在随机的特征子集上寻找最优划分特征，进一步增加了划分的随机性。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=500, n_jobs=-1,\n",
       "            oob_score=True, random_state=666, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "rf_clf = RandomForestClassifier(n_estimators=500, random_state=666, \n",
    "                                oob_score=True, n_jobs=-1)\n",
    "rf_clf.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.892"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_clf.oob_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "修改随机森林的最大叶子节点数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=16,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=500, n_jobs=-1,\n",
       "            oob_score=True, random_state=666, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_clf2 = RandomForestClassifier(n_estimators=500, random_state=666, \n",
    "                                oob_score=True, n_jobs=-1, max_leaf_nodes=16)\n",
    "rf_clf2.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.906"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_clf2.oob_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 使用另外一种随机森林（Extra-Trees)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Extra-Trees：Extramely Random Trees，极其随机的森林\n",
    "\n",
    "底层使用的决策树在节点划分上，使用随机的特征和随机的阈值。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ExtraTreesClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "           max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "           min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "           min_samples_leaf=1, min_samples_split=2,\n",
       "           min_weight_fraction_leaf=0.0, n_estimators=500, n_jobs=None,\n",
       "           oob_score=True, random_state=666, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "\n",
    "et_clf = ExtraTreesClassifier(n_estimators=500, bootstrap=True, oob_score=True, random_state=666)\n",
    "et_clf.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.892"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "et_clf.oob_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ada Boosting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = datasets.make_moons(n_samples=500, noise=0.3, random_state=42)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 使用sklearn中的AdaBoostClassifier进行分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AdaBoostClassifier(algorithm='SAMME.R',\n",
       "          base_estimator=DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=2,\n",
       "            max_features=None, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, presort=False, random_state=None,\n",
       "            splitter='best'),\n",
       "          learning_rate=1.0, n_estimators=500, random_state=None)"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "\n",
    "ada_clf = AdaBoostClassifier(DecisionTreeClassifier(max_depth=2), n_estimators=500)\n",
    "ada_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.872"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ada_clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gradient Boosting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 使用sklearn中的AdaBoostClassifier进行分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GradientBoostingClassifier(criterion='friedman_mse', init=None,\n",
       "              learning_rate=0.1, loss='deviance', max_depth=2,\n",
       "              max_features=None, max_leaf_nodes=None,\n",
       "              min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "              min_samples_leaf=1, min_samples_split=2,\n",
       "              min_weight_fraction_leaf=0.0, n_estimators=30,\n",
       "              n_iter_no_change=None, presort='auto', random_state=None,\n",
       "              subsample=1.0, tol=0.0001, validation_fraction=0.1,\n",
       "              verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "\n",
    "gb_clf = GradientBoostingClassifier(max_depth=2, n_estimators=30)\n",
    "gb_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## stacking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.832"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gb_clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 集成学习解决回归问题"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import BaggingRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.ensemble import ExtraTreesRegressor\n",
    "from sklearn.ensemble import AdaBoostRegressor\n",
    "from sklearn.ensemble import GradientBoostingRegressor"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "307.448px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
